From f445f512e5fcb3e12703c3fceea4872bc59351ba Mon Sep 17 00:00:00 2001 From: psteinroe Date: Fri, 10 Apr 2026 10:18:28 +0200 Subject: [PATCH] initial --- .../data/multi/20220117141357_extensions.sql | 4 + .../data/multi/20220117141359_app_schema.sql | 5 + .../data/multi/20220117141507_semver.sql | 68 + .../multi/20220117141645_valid_name_type.sql | 29 + .../20220117141942_email_address_type.sql | 5 + .../20220117142104_account_and_org_tables.sql | 138 ++ .../multi/20220117142137_package_tables.sql | 65 + .../multi/20220117142138_developer_tools.sql | 28 + .../20220117142141_security_utilities.sql | 99 + .../20220117142142_security_definitions.sql | 195 ++ .../tests/data/multi/20220117155720_views.sql | 88 + .../tests/data/multi/20220117155820_rpc.sql | 149 ++ .../20230323180034_reserved_user_accts.sql | 82 + .../20230328185043_olirice_asciiplot.sql | 162 ++ .../multi/20230330155137_supabase_dbdev.sql | 284 +++ .../20230331145934_burggraf-pg_headerkit.sql | 268 +++ .../20230331163908_olirice-index_advisor.sql | 324 ++++ .../20230331163909_olirice-read_once.sql | 161 ++ .../20230404162614_michelp-adminpack.sql | 1564 ++++++++++++++++ .../20230405083103_fix_auth_schema_values.sql | 30 + .../20230405085810_fix_avatars_handle.sql | 41 + .../multi/20230405163940_download_metrics.sql | 93 + ...448_download_metrics_computed_relation.sql | 8 + ...30411175952_langchain-embedding_search.sql | 139 ++ ...20230411175953_langchain-hybrid_search.sql | 160 ++ ...230413130634_popular_packages_function.sql | 11 + ...20230413140356_update_profile_function.sql | 22 + .../20230417141004_dbdev_short_desc_typo.sql | 4 + .../20230508165641_packages_order_version.sql | 20 + ...75952_langchain-embedding_search-1_1_0.sql | 167 ++ ...08175953_langchain-hybrid_search-1_1_0.sql | 144 ++ ...212339_langchain_headerkit_config_dump.sql | 37 + ...81432_dbdev_supports_multiple_versions.sql | 284 +++ .../20230829125510_fix_view_permissions.sql | 6 + ...0830083255_olirice-index_advisor-0_2_0.sql | 343 ++++ ...915_allow_anon_access_to_package_views.sql | 3 + .../multi/20230906110845_access_token.sql | 196 ++ .../multi/20230906111353_publish_package.sql | 106 ++ ...ow_publishing_relocatable_and_requires.sql | 167 ++ .../20231205051816_add_default_version.sql | 95 + ...5101809_dbdev_supports_default_version.sql | 340 ++++ .../multi/20231207071422_new_package_name.sql | 82 + ...73048_dbdev_supports_new_package_names.sql | 342 ++++ ...11703_langchain@embedding_search-1.1.1.sql | 167 ++ ...07112129_langchain@hybrid_search-1.1.1.sql | 144 ++ ...20231207112942_michelp@adminpack-0.0.2.sql | 1555 ++++++++++++++++ ...1207113329_olirice@index_advisor-0.2.1.sql | 343 ++++ ...20231207113857_olirice@read_once-0.3.2.sql | 145 ++ .../20240108072747_update_provider_id.sql | 8 + .../20240605122023_fix_view_permissions.sql | 26 + .../20240705083738_remove_contact_email.sql | 33 + .../20250106073735_jwt_secret_from_vault.sql | 66 + ...50217100252_restrict_accounts_and_orgs.sql | 49 + ...152_remove_dbdev_from_popular_packages.sql | 12 + .../tests__20220117141357_extensions_100.snap | 11 + .../tests__20220117141357_extensions_80.snap | 11 + .../tests__20220117141359_app_schema_100.snap | 9 + .../tests__20220117141359_app_schema_80.snap | 14 + .../tests__20220117141507_semver_100.snap | 60 + .../tests__20220117141507_semver_80.snap | 64 + ...s__20220117141645_valid_name_type_100.snap | 16 + ...ts__20220117141645_valid_name_type_80.snap | 17 + ...20220117141942_email_address_type_100.snap | 7 + ..._20220117141942_email_address_type_80.snap | 7 + ...0117142104_account_and_org_tables_100.snap | 162 ++ ...20117142104_account_and_org_tables_80.snap | 172 ++ ...ts__20220117142137_package_tables_100.snap | 104 ++ ...sts__20220117142137_package_tables_80.snap | 119 ++ ...s__20220117142138_developer_tools_100.snap | 32 + ...ts__20220117142138_developer_tools_80.snap | 32 + ...20220117142141_security_utilities_100.snap | 98 + ..._20220117142141_security_utilities_80.snap | 106 ++ ...220117142142_security_definitions_100.snap | 196 ++ ...0220117142142_security_definitions_80.snap | 226 +++ .../tests__20220117155720_views_100.snap | 99 + .../multi/tests__20220117155720_views_80.snap | 99 + .../multi/tests__20220117155820_rpc_100.snap | 23 + .../multi/tests__20220117155820_rpc_80.snap | 23 + ...0230323180034_reserved_user_accts_100.snap | 145 ++ ...20230323180034_reserved_user_accts_80.snap | 145 ++ ..._20230328185043_olirice_asciiplot_100.snap | 191 ++ ...__20230328185043_olirice_asciiplot_80.snap | 196 ++ ...ts__20230330155137_supabase_dbdev_100.snap | 301 ++++ ...sts__20230330155137_supabase_dbdev_80.snap | 306 ++++ ...30331145934_burggraf-pg_headerkit_100.snap | 277 +++ ...230331145934_burggraf-pg_headerkit_80.snap | 282 +++ ...30331163908_olirice-index_advisor_100.snap | 334 ++++ ...230331163908_olirice-index_advisor_80.snap | 339 ++++ ..._20230331163909_olirice-read_once_100.snap | 172 ++ ...__20230331163909_olirice-read_once_80.snap | 177 ++ ..._20230404162614_michelp-adminpack_100.snap | 1582 ++++++++++++++++ ...__20230404162614_michelp-adminpack_80.snap | 1587 +++++++++++++++++ ...0405083103_fix_auth_schema_values_100.snap | 33 + ...30405083103_fix_auth_schema_values_80.snap | 38 + ...20230405085810_fix_avatars_handle_100.snap | 44 + ..._20230405085810_fix_avatars_handle_80.snap | 44 + ...__20230405163940_download_metrics_100.snap | 102 ++ ...s__20230405163940_download_metrics_80.snap | 109 ++ ...ownload_metrics_computed_relation_100.snap | 14 + ...download_metrics_computed_relation_80.snap | 14 + ...175952_langchain-embedding_search_100.snap | 148 ++ ...1175952_langchain-embedding_search_80.snap | 154 ++ ...411175953_langchain-hybrid_search_100.snap | 169 ++ ...0411175953_langchain-hybrid_search_80.snap | 174 ++ ...3130634_popular_packages_function_100.snap | 16 + ...13130634_popular_packages_function_80.snap | 16 + ...413140356_update_profile_function_100.snap | 27 + ...0413140356_update_profile_function_80.snap | 27 + ...30417141004_dbdev_short_desc_typo_100.snap | 8 + ...230417141004_dbdev_short_desc_typo_80.snap | 9 + ...0508165641_packages_order_version_100.snap | 28 + ...30508165641_packages_order_version_80.snap | 28 + ..._langchain-embedding_search-1_1_0_100.snap | 178 ++ ...2_langchain-embedding_search-1_1_0_80.snap | 184 ++ ...953_langchain-hybrid_search-1_1_0_100.snap | 155 ++ ...5953_langchain-hybrid_search-1_1_0_80.snap | 160 ++ ...9_langchain_headerkit_config_dump_100.snap | 79 + ...39_langchain_headerkit_config_dump_80.snap | 82 + ..._dbdev_supports_multiple_versions_100.snap | 296 +++ ...2_dbdev_supports_multiple_versions_80.snap | 301 ++++ ...230829125510_fix_view_permissions_100.snap | 21 + ...0230829125510_fix_view_permissions_80.snap | 21 + ...83255_olirice-index_advisor-0_2_0_100.snap | 354 ++++ ...083255_olirice-index_advisor-0_2_0_80.snap | 359 ++++ ...llow_anon_access_to_package_views_100.snap | 12 + ...allow_anon_access_to_package_views_80.snap | 12 + ...ests__20230906110845_access_token_100.snap | 201 +++ ...tests__20230906110845_access_token_80.snap | 204 +++ ...s__20230906111353_publish_package_100.snap | 109 ++ ...ts__20230906111353_publish_package_80.snap | 116 ++ ...blishing_relocatable_and_requires_100.snap | 171 ++ ...ublishing_relocatable_and_requires_80.snap | 171 ++ ...0231205051816_add_default_version_100.snap | 109 ++ ...20231205051816_add_default_version_80.snap | 111 ++ ...09_dbdev_supports_default_version_100.snap | 352 ++++ ...809_dbdev_supports_default_version_80.snap | 357 ++++ ...__20231207071422_new_package_name_100.snap | 88 + ...s__20231207071422_new_package_name_80.snap | 92 + ..._dbdev_supports_new_package_names_100.snap | 354 ++++ ...8_dbdev_supports_new_package_names_80.snap | 359 ++++ ..._langchain@embedding_search-1.1.1_100.snap | 178 ++ ...3_langchain@embedding_search-1.1.1_80.snap | 184 ++ ...129_langchain@hybrid_search-1.1.1_100.snap | 155 ++ ...2129_langchain@hybrid_search-1.1.1_80.snap | 161 ++ ...207112942_michelp@adminpack-0.0.2_100.snap | 1566 ++++++++++++++++ ...1207112942_michelp@adminpack-0.0.2_80.snap | 1571 ++++++++++++++++ ...13329_olirice@index_advisor-0.2.1_100.snap | 354 ++++ ...113329_olirice@index_advisor-0.2.1_80.snap | 359 ++++ ...207113857_olirice@read_once-0.3.2_100.snap | 156 ++ ...1207113857_olirice@read_once-0.3.2_80.snap | 161 ++ ...20240108072747_update_provider_id_100.snap | 5 + ..._20240108072747_update_provider_id_80.snap | 5 + ...240605122023_fix_view_permissions_100.snap | 33 + ...0240605122023_fix_view_permissions_80.snap | 33 + ...240705083738_remove_contact_email_100.snap | 37 + ...0240705083738_remove_contact_email_80.snap | 37 + ...50106073735_jwt_secret_from_vault_100.snap | 65 + ...250106073735_jwt_secret_from_vault_80.snap | 65 + ...100252_restrict_accounts_and_orgs_100.snap | 42 + ...7100252_restrict_accounts_and_orgs_80.snap | 48 + ...emove_dbdev_from_popular_packages_100.snap | 17 + ...remove_dbdev_from_popular_packages_80.snap | 17 + 162 files changed, 28306 insertions(+) create mode 100644 crates/pgls_pretty_print/tests/data/multi/20220117141357_extensions.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20220117141359_app_schema.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20220117141507_semver.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20220117141645_valid_name_type.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20220117141942_email_address_type.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20220117142104_account_and_org_tables.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20220117142137_package_tables.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20220117142138_developer_tools.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20220117142141_security_utilities.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20220117142142_security_definitions.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20220117155720_views.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20220117155820_rpc.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230323180034_reserved_user_accts.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230328185043_olirice_asciiplot.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230330155137_supabase_dbdev.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230331145934_burggraf-pg_headerkit.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230331163908_olirice-index_advisor.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230331163909_olirice-read_once.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230404162614_michelp-adminpack.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230405083103_fix_auth_schema_values.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230405085810_fix_avatars_handle.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230405163940_download_metrics.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230411104448_download_metrics_computed_relation.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230411175952_langchain-embedding_search.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230411175953_langchain-hybrid_search.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230413130634_popular_packages_function.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230413140356_update_profile_function.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230417141004_dbdev_short_desc_typo.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230508165641_packages_order_version.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230508175952_langchain-embedding_search-1_1_0.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230508175953_langchain-hybrid_search-1_1_0.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230622212339_langchain_headerkit_config_dump.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230623181432_dbdev_supports_multiple_versions.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230829125510_fix_view_permissions.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230830083255_olirice-index_advisor-0_2_0.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230831172915_allow_anon_access_to_package_views.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230906110845_access_token.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20230906111353_publish_package.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20231110061036_allow_publishing_relocatable_and_requires.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20231205051816_add_default_version.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20231205101809_dbdev_supports_default_version.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20231207071422_new_package_name.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20231207073048_dbdev_supports_new_package_names.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20231207111703_langchain@embedding_search-1.1.1.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20231207112129_langchain@hybrid_search-1.1.1.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20231207112942_michelp@adminpack-0.0.2.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20231207113329_olirice@index_advisor-0.2.1.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20231207113857_olirice@read_once-0.3.2.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20240108072747_update_provider_id.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20240605122023_fix_view_permissions.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20240705083738_remove_contact_email.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20250106073735_jwt_secret_from_vault.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20250217100252_restrict_accounts_and_orgs.sql create mode 100644 crates/pgls_pretty_print/tests/data/multi/20250804111152_remove_dbdev_from_popular_packages.sql create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141357_extensions_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141357_extensions_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141359_app_schema_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141359_app_schema_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141507_semver_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141507_semver_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141645_valid_name_type_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141645_valid_name_type_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141942_email_address_type_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141942_email_address_type_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142104_account_and_org_tables_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142104_account_and_org_tables_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142137_package_tables_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142137_package_tables_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142138_developer_tools_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142138_developer_tools_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142141_security_utilities_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142141_security_utilities_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142142_security_definitions_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142142_security_definitions_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155720_views_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155720_views_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155820_rpc_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155820_rpc_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230323180034_reserved_user_accts_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230323180034_reserved_user_accts_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230328185043_olirice_asciiplot_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230328185043_olirice_asciiplot_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230330155137_supabase_dbdev_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230330155137_supabase_dbdev_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331145934_burggraf-pg_headerkit_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331145934_burggraf-pg_headerkit_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163908_olirice-index_advisor_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163908_olirice-index_advisor_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163909_olirice-read_once_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163909_olirice-read_once_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230404162614_michelp-adminpack_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230404162614_michelp-adminpack_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405083103_fix_auth_schema_values_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405083103_fix_auth_schema_values_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405085810_fix_avatars_handle_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405085810_fix_avatars_handle_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405163940_download_metrics_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405163940_download_metrics_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411104448_download_metrics_computed_relation_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411104448_download_metrics_computed_relation_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175952_langchain-embedding_search_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175952_langchain-embedding_search_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175953_langchain-hybrid_search_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175953_langchain-hybrid_search_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413130634_popular_packages_function_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413130634_popular_packages_function_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413140356_update_profile_function_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413140356_update_profile_function_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230417141004_dbdev_short_desc_typo_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230417141004_dbdev_short_desc_typo_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508165641_packages_order_version_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508165641_packages_order_version_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175952_langchain-embedding_search-1_1_0_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175952_langchain-embedding_search-1_1_0_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175953_langchain-hybrid_search-1_1_0_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175953_langchain-hybrid_search-1_1_0_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230622212339_langchain_headerkit_config_dump_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230622212339_langchain_headerkit_config_dump_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230623181432_dbdev_supports_multiple_versions_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230623181432_dbdev_supports_multiple_versions_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230829125510_fix_view_permissions_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230829125510_fix_view_permissions_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230830083255_olirice-index_advisor-0_2_0_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230830083255_olirice-index_advisor-0_2_0_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230831172915_allow_anon_access_to_package_views_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230831172915_allow_anon_access_to_package_views_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906110845_access_token_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906110845_access_token_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906111353_publish_package_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906111353_publish_package_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231110061036_allow_publishing_relocatable_and_requires_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231110061036_allow_publishing_relocatable_and_requires_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205051816_add_default_version_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205051816_add_default_version_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205101809_dbdev_supports_default_version_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205101809_dbdev_supports_default_version_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207071422_new_package_name_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207071422_new_package_name_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207073048_dbdev_supports_new_package_names_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207073048_dbdev_supports_new_package_names_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207111703_langchain@embedding_search-1.1.1_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207111703_langchain@embedding_search-1.1.1_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112129_langchain@hybrid_search-1.1.1_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112129_langchain@hybrid_search-1.1.1_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112942_michelp@adminpack-0.0.2_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112942_michelp@adminpack-0.0.2_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113329_olirice@index_advisor-0.2.1_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113329_olirice@index_advisor-0.2.1_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113857_olirice@read_once-0.3.2_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113857_olirice@read_once-0.3.2_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20240108072747_update_provider_id_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20240108072747_update_provider_id_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20240605122023_fix_view_permissions_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20240605122023_fix_view_permissions_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20240705083738_remove_contact_email_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20240705083738_remove_contact_email_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20250106073735_jwt_secret_from_vault_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20250106073735_jwt_secret_from_vault_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20250217100252_restrict_accounts_and_orgs_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20250217100252_restrict_accounts_and_orgs_80.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20250804111152_remove_dbdev_from_popular_packages_100.snap create mode 100644 crates/pgls_pretty_print/tests/snapshots/multi/tests__20250804111152_remove_dbdev_from_popular_packages_80.snap diff --git a/crates/pgls_pretty_print/tests/data/multi/20220117141357_extensions.sql b/crates/pgls_pretty_print/tests/data/multi/20220117141357_extensions.sql new file mode 100644 index 000000000..6bd934c8c --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20220117141357_extensions.sql @@ -0,0 +1,4 @@ +create extension if not exists pg_stat_statements with schema extensions; +create extension if not exists pg_trgm with schema extensions; +create extension if not exists citext with schema extensions; +create extension if not exists pg_cron; diff --git a/crates/pgls_pretty_print/tests/data/multi/20220117141359_app_schema.sql b/crates/pgls_pretty_print/tests/data/multi/20220117141359_app_schema.sql new file mode 100644 index 000000000..eaee13bc5 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20220117141359_app_schema.sql @@ -0,0 +1,5 @@ +create schema app; + +grant usage on schema app to authenticated, anon; + +alter default privileges in schema app grant select on tables to authenticated, anon; diff --git a/crates/pgls_pretty_print/tests/data/multi/20220117141507_semver.sql b/crates/pgls_pretty_print/tests/data/multi/20220117141507_semver.sql new file mode 100644 index 000000000..761a40543 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20220117141507_semver.sql @@ -0,0 +1,68 @@ +-- https://semver.org/#backusnaur-form-grammar-for-valid-semver-versions +create type app.semver_struct as ( + major smallint, + minor smallint, + patch smallint +); + +create or replace function app.is_valid(app.semver_struct) + returns boolean + immutable + language sql +as $$ + select ( + ($1).major is not null + and ($1).minor is not null + and ($1).patch is not null + ) +$$; + +create domain app.semver + as app.semver_struct + check ( + app.is_valid(value) +); + +create function app.semver_exception(version text) + returns app.semver_struct + immutable + language plpgsql +as $$ +begin + raise exception using errcode='22000', message=format('Invalid semver %L', version); +end; +$$; + + +-- Cast from Text +create function app.text_to_semver(text) + returns app.semver_struct + immutable + strict + language sql +as $$ + with s(version) as ( + select ( + split_part($1, '.', 1), + split_part($1, '.', 2), + split_part(split_part(split_part($1, '.', 3), '-', 1), '+', 1) + )::app.semver_struct + ) + select + case app.is_valid(s.version) + when true then s.version + else app.semver_exception($1) + end + from + s +$$; + + +create or replace function app.semver_to_text(app.semver) + returns text + immutable + language sql +as $$ + select + format('%s.%s.%s', $1.major, $1.minor, $1.patch) +$$; diff --git a/crates/pgls_pretty_print/tests/data/multi/20220117141645_valid_name_type.sql b/crates/pgls_pretty_print/tests/data/multi/20220117141645_valid_name_type.sql new file mode 100644 index 000000000..849e98314 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20220117141645_valid_name_type.sql @@ -0,0 +1,29 @@ +create extension if not exists citext with schema extensions; + +create domain app.valid_name + as extensions.citext + check ( + -- 3 to 15 chars, A-z with underscores + value ~ '^[A-z][A-z0-9\_]{2,32}$' +); + +create or replace function app.exception(message text) + returns text + language plpgsql + as $$ + begin + raise exception using errcode='22000', message=message; + end; + $$; + +/* +create domain app.valid_name + as extensions.citext + check ( + -- 3 to 15 chars, A-z with underscores + case + when value ~ '^[A-z][A-z0-9\_]{2,14}$' then True + else app.exception('Bad name ' || value)::bool + end +); +*/ diff --git a/crates/pgls_pretty_print/tests/data/multi/20220117141942_email_address_type.sql b/crates/pgls_pretty_print/tests/data/multi/20220117141942_email_address_type.sql new file mode 100644 index 000000000..951cd1fda --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20220117141942_email_address_type.sql @@ -0,0 +1,5 @@ +create domain app.email_address + -- https://html.spec.whatwg.org/multipage/input.html#email-state-(type=email) + AS citext + check ( value ~ '^[a-zA-Z0-9.!#$%&''*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$' +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20220117142104_account_and_org_tables.sql b/crates/pgls_pretty_print/tests/data/multi/20220117142104_account_and_org_tables.sql new file mode 100644 index 000000000..0dbfeca14 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20220117142104_account_and_org_tables.sql @@ -0,0 +1,138 @@ +insert into storage.buckets ("id", "name") +values ('avatars', 'avatars'); + +create table app.handle_registry( + /* + Enforces uniqueness of handles across orgs and accounts + e.g. jsmith or supabase + */ + handle app.valid_name primary key not null, + is_organization boolean not null, + created_at timestamptz not null default now(), + unique (handle, is_organization) +); + +create table app.accounts( + -- 1:1 with auth.users + id uuid primary key references auth.users(id), + handle app.valid_name not null unique, + is_organization boolean generated always as (false) stored, + avatar_id uuid references storage.objects(id), + display_name text check (length(display_name) <= 128), + bio text check (length(bio) <= 512), + contact_email app.email_address, + created_at timestamptz not null default now(), + + constraint fk_handle_registry + foreign key (handle, is_organization) + references app.handle_registry(handle, is_organization) +); + +create or replace function app.register_account() + returns trigger + language plpgsql + security definer + as $$ + begin + insert into app.handle_registry (handle, is_organization) + values ( + new.raw_user_meta_data ->> 'handle', + false + ); + + insert into app.accounts (id, handle, display_name, bio, contact_email) + values ( + new.id, + new.raw_user_meta_data ->> 'handle', + new.raw_user_meta_data ->> 'display_name', + new.raw_user_meta_data ->> 'bio', + new.raw_user_meta_data ->> 'contact_email' + ); + return new; + end; + $$; + +create or replace trigger on_auth_user_created + after insert on auth.users + for each row execute procedure app.register_account(); + +create table app.organizations( + id uuid primary key default gen_random_uuid(), + handle app.valid_name not null unique, + is_organization boolean generated always as (true) stored, + avatar_id uuid references storage.objects(id), + display_name text check (length(display_name) <= 128), + bio text check (length(bio) <= 512), + contact_email app.email_address, + -- enforced so organization always have at least 1 admin member + created_at timestamptz not null default now(), + + constraint fk_handle_registry + foreign key (handle, is_organization) + references app.handle_registry(handle, is_organization) +); + +create type app.membership_role as enum ('maintainer'); + +create table app.members( + id uuid primary key default uuid_generate_v4(), + organization_id uuid not null references app.organizations(id), + account_id uuid not null references app.accounts(id), + role app.membership_role not null, + created_at timestamptz not null default now(), + unique (organization_id, account_id) +); + +create or replace function app.register_organization_creator_as_member() + returns trigger + language plpgsql + security definer + as $$ + begin + insert into app.members(organization_id, account_id, role) + values (new.id, auth.uid(), 'maintainer'); + + return new; + end; + $$; + +create or replace trigger on_app_organization_created + after insert on app.organizations + for each row execute procedure app.register_organization_creator_as_member(); + +create or replace function app.update_avatar_id() + returns trigger + language plpgsql + security definer + as $$ + declare + v_handle app.valid_name; + v_affected_account app.accounts := null; + begin + select (string_to_array(new.name, '-'::text))[1]::app.valid_name into v_handle; + + update app.accounts + set avatar_id = new.id + where handle = v_handle + returning * into v_affected_account; + + if not v_affected_account is null then + update auth.users u + set + "raw_user_meta_data" = u.raw_user_meta_data || jsonb_build_object( + 'avatar_path', new.name + ) + where u.id = v_affected_account.id; + else + update app.organizations + set avatar_id = new.id + where handle = v_handle; + end if; + + return new; + end; + $$; + +create or replace trigger on_storage_object_created + after insert on storage.objects + for each row when(new.bucket_id = 'avatars') execute procedure app.update_avatar_id(); diff --git a/crates/pgls_pretty_print/tests/data/multi/20220117142137_package_tables.sql b/crates/pgls_pretty_print/tests/data/multi/20220117142137_package_tables.sql new file mode 100644 index 000000000..717ff015a --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20220117142137_package_tables.sql @@ -0,0 +1,65 @@ +insert into storage.buckets (id, name) +values + ('package_versions', 'package_versions'), + ('package_upgrades', 'package_upgrades'); + +create function app.to_package_name(handle app.valid_name, partial_name app.valid_name) + returns text + immutable + language sql +as $$ + select format('%s-%s', $1, $2) +$$; + +create table app.packages( + id uuid primary key default gen_random_uuid(), + package_name text not null generated always as (app.to_package_name(handle, partial_name)) stored, + handle app.valid_name not null references app.handle_registry(handle), + partial_name app.valid_name not null, -- ex: math + control_description varchar(1000), + control_relocatable bool not null default false, + control_requires varchar(128)[] default '{}'::varchar(128)[], + created_at timestamptz not null default now(), + unique (handle, partial_name) +); +create index packages_partial_name_search_idx on app.packages using gin (partial_name extensions.gin_trgm_ops); +create index packages_handle_search_idx on app.packages using gin (handle extensions.gin_trgm_ops); + +create table app.package_versions( + id uuid primary key default gen_random_uuid(), + package_id uuid not null references app.packages(id), + version_struct app.semver not null, + version text not null generated always as (app.semver_to_text(version_struct)) stored, + sql varchar(250000), + description_md varchar(250000), + created_at timestamptz not null default now(), + unique(package_id, version_struct) +); + +create table app.package_upgrades( + id uuid primary key default gen_random_uuid(), + package_id uuid not null references app.packages(id), + from_version_struct app.semver not null, + from_version text not null generated always as (app.semver_to_text(from_version_struct)) stored, + to_version_struct app.semver not null, + to_version text not null generated always as (app.semver_to_text(to_version_struct)) stored, + sql varchar(250000), + created_at timestamptz not null default now(), + unique(package_id, from_version_struct, to_version_struct) +); + +create function app.version_text_to_handle(version text) + returns app.valid_name + immutable + language sql +as $$ + select split_part($1, '-', 1) +$$; + +create function app.version_text_to_package_partial_name(version text) + returns app.valid_name + immutable + language sql +as $$ + select split_part($1, '--', 2) +$$; diff --git a/crates/pgls_pretty_print/tests/data/multi/20220117142138_developer_tools.sql b/crates/pgls_pretty_print/tests/data/multi/20220117142138_developer_tools.sql new file mode 100644 index 000000000..22cf2baad --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20220117142138_developer_tools.sql @@ -0,0 +1,28 @@ +create or replace function app.simulate_login(email citext) + returns void + language sql +as $$ + /* + Simulated JWT of logged in user + */ + + select + set_config( + 'request.jwt.claims', + ( + select + json_build_object( + 'sub', + id, + 'role', + 'authenticated' + )::text + from + auth.users + where + email = $1 + ), + true + ), + set_config('role', 'authenticated', true) +$$; diff --git a/crates/pgls_pretty_print/tests/data/multi/20220117142141_security_utilities.sql b/crates/pgls_pretty_print/tests/data/multi/20220117142141_security_utilities.sql new file mode 100644 index 000000000..eb81df4da --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20220117142141_security_utilities.sql @@ -0,0 +1,99 @@ +create function app.is_organization_maintainer(account_id uuid, organization_id uuid) + returns boolean + language sql + stable +as $$ + -- Does the currently authenticated user have permission to admin orgs and org members? + select + exists( + select + 1 + from + app.members m + where + m.account_id = $1 + and m.organization_id = $2 + and m.role = 'maintainer' + ) +$$; + + +create function app.is_handle_maintainer(account_id uuid, handle app.valid_name) + returns boolean + language sql + stable +as $$ + select + exists( + select + 1 + from + app.accounts acc + where + acc.id = $1 + and acc.handle = $2 + ) + or exists( + select + 1 + from + app.organizations o + join app.members m + on o.id = m.organization_id + where + m.role = 'maintainer' + and m.account_id = $1 + and o.handle = $2 + ) +$$; + + + + +create function app.is_package_maintainer(account_id uuid, package_id uuid) + returns boolean + language sql + stable +as $$ + select + exists( + select + 1 + from + app.accounts acc + join app.packages p + on acc.handle = p.handle + where + acc.id = $1 + and p.id = $2 + ) + or exists( + -- current user is maintainer of org that owns the package + select + 1 + from + app.packages p + join app.organizations o + on p.handle = o.handle + join app.members m + on o.id = m.organization_id + where + m.role = 'maintainer' + and m.account_id = $1 + and p.id = $2 + ) +$$; + + +create function app.is_package_version_maintainer(account_id uuid, package_version_id uuid) + returns boolean + language sql + stable +as $$ + select + app.is_package_maintainer($1, pv.package_id) + from + app.package_versions pv + where + pv.id = $2 +$$; diff --git a/crates/pgls_pretty_print/tests/data/multi/20220117142142_security_definitions.sql b/crates/pgls_pretty_print/tests/data/multi/20220117142142_security_definitions.sql new file mode 100644 index 000000000..0d2d41941 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20220117142142_security_definitions.sql @@ -0,0 +1,195 @@ +-- app.handle_registry +grant insert + (handle, is_organization) + on app.handle_registry + to authenticated; + + +-- app.accounts +alter table app.accounts enable row level security; + +grant update + (avatar_id, display_name, bio, contact_email) + on app.accounts + to authenticated; + +create policy accounts_update_policy + on app.accounts + as permissive + for update + to authenticated + using (id = auth.uid()); + +create policy accounts_select_policy + on app.accounts + as permissive + for select + to authenticated + using (true); + +-- app.organizations +alter table app.organizations enable row level security; + +grant insert + (handle, avatar_id, display_name, bio, contact_email) + on app.organizations + to authenticated; + +grant update + (avatar_id, display_name, bio, contact_email) + on app.organizations + to authenticated; + +create policy organizations_insert_policy + on app.organizations + as permissive + for insert + to authenticated + with check (true); + +create policy organizations_update_policy + on app.organizations + as permissive + for update + to authenticated + using (app.is_organization_maintainer(auth.uid(), id)); + +create policy organizations_select_policy + on app.organizations + as permissive + for select + to authenticated + using (true); + +-- app.members +alter table app.members enable row level security; + +grant insert + (organization_id, account_id, role) + on app.members + to authenticated; + +grant delete + on app.members + to authenticated; + +create policy members_insert_policy + on app.members + as permissive + for insert + to authenticated + with check (app.is_organization_maintainer(auth.uid(), organization_id)); + +create policy members_delete_policy + on app.members + as permissive + for delete + to authenticated + using (app.is_organization_maintainer(auth.uid(), organization_id)); + +create policy members_select_policy + on app.members + as permissive + for select + to authenticated + using (true); + +-- app.packages +alter table app.packages enable row level security; + +grant insert (partial_name, handle) + on app.packages + to authenticated; + +create policy package_insert_policy + on app.packages + as permissive + for insert + to authenticated + with check (app.is_handle_maintainer(auth.uid(), handle)); + +create policy packages_select_policy + on app.packages + as permissive + for select + to authenticated + using (true); + +-- app.package_versions +alter table app.package_versions enable row level security; + +grant insert + (package_id, version_struct, sql, description_md) + on app.package_versions + to authenticated; + +create policy package_versions_insert_policy + on app.package_versions + as permissive + for insert + to authenticated + with check ( app.is_package_maintainer(auth.uid(), package_id) ); + +create policy package_versions_update_policy + on app.package_versions + as permissive + for update + to authenticated + using ( app.is_package_maintainer(auth.uid(), package_id) ); + +create policy package_versions_select_policy + on app.package_versions + as permissive + for select + to public + using (true); + +-- app.package_upgrades +alter table app.package_upgrades enable row level security; + +grant insert + (package_id, from_version_struct, to_version_struct, sql) + on app.package_upgrades + to authenticated; + +create policy package_upgrades_insert_policy + on app.package_upgrades + as permissive + for insert + to authenticated + with check ( app.is_package_maintainer(auth.uid(), package_id) ); + +create policy package_upgrades_update_policy + on app.package_upgrades + as permissive + for update + to authenticated + using ( app.is_package_maintainer(auth.uid(), package_id) ); + +create policy package_upgrades_select_policy + on app.package_upgrades + as permissive + for select + to public + using (true); + +-- storage.objects + +create policy storage_objects_insert_policy + on storage.objects + as permissive + for insert + to authenticated + with check ( + app.is_handle_maintainer( + auth.uid(), + (string_to_array(name, '-'::text))[1]::app.valid_name + ) + ); + +create policy storage_objects_select_policy + on storage.objects + as permissive + for select + to public -- all roles + using (bucket_id = 'avatars'); diff --git a/crates/pgls_pretty_print/tests/data/multi/20220117155720_views.sql b/crates/pgls_pretty_print/tests/data/multi/20220117155720_views.sql new file mode 100644 index 000000000..66bb695b6 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20220117155720_views.sql @@ -0,0 +1,88 @@ +create view public.accounts as + select + acc.id, + acc.handle, + obj.name as avatar_path, + acc.display_name, + acc.bio, + acc.contact_email, + acc.created_at + from + app.accounts acc + left join storage.objects obj + on acc.avatar_id = obj.id; + +create view public.organizations as + select + org.id, + org.handle, + obj.name as avatar_path, + org.display_name, + org.bio, + org.contact_email, + org.created_at + from + app.organizations org + left join storage.objects obj + on org.avatar_id = obj.id; + +create view public.members as + select + aio.organization_id, + aio.account_id, + aio.role, + aio.created_at + from + app.members aio; + +create view public.packages as + select + pa.id, + pa.package_name, + pa.handle, + pa.partial_name, + newest_ver.version as latest_version, + newest_ver.description_md, + pa.control_description, + pa.control_requires, + pa.created_at + from + app.packages pa, + lateral ( + select * + from app.package_versions pv + where pv.package_id = pa.id + order by pv.version_struct + limit 1 + ) newest_ver; + +create view public.package_versions as + select + pv.id, + pv.package_id, + pa.package_name, + pv.version, + pv.sql, + pv.description_md, + pa.control_description, + pa.control_requires, + pv.created_at + from + app.packages pa + join app.package_versions pv + on pa.id = pv.package_id; + +create view public.package_upgrades + as + select + pu.id, + pu.package_id, + pa.package_name, + pu.from_version, + pu.to_version, + pu.sql, + pu.created_at + from + app.packages pa + join app.package_upgrades pu + on pa.id = pu.package_id; diff --git a/crates/pgls_pretty_print/tests/data/multi/20220117155820_rpc.sql b/crates/pgls_pretty_print/tests/data/multi/20220117155820_rpc.sql new file mode 100644 index 000000000..31b10f611 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20220117155820_rpc.sql @@ -0,0 +1,149 @@ +/* +create function public.create_organization( + handle app.valid_name, + display_name text = null, + bio text = null, + contact_email citext = null, + avatar_id uuid = null +) + returns public.organizations + language plpgsql +as $$ +begin + -- Register the requested handle + insert into app.handle_registry(handle, is_organization) values ($1, true); + + -- Create the organization + insert into app.organizations(handle, display_name, bio, contact_email, avatar_id) + values ($1, $2, $3, $4, $5); + + -- Return the org + return org from public.organizations org where org.handle = $1; +end; +$$; + +create function public.publish_package_version( + handle app.valid_name, + package_partial_name app.valid_name, + version text, + description_md text, + sql text +) + returns public.package_versions + language plpgsql +as $$ +declare + acc app.accounts = acc from app.accounts acc where id = auth.uid(); + package_id uuid; + package_version_id uuid; +begin + -- Upsert package + -- TODO add description or markdown object + insert into app.packages(partial_name, handle, description_md) + values (package_partial_name, package_handle) + on conflict do update + set description_md = excluded.description_md + returning id + into package_id; + + -- Insert package_version + insert into app.package_versions(package_id, version_struct, sql) + values ( + package_id, + app.text_to_semver(version), + sql + ) + returning id + into package_version_id; + + -- Return the package version + return pv from public.package_versions pv where pv.id = package_version_id; +end; +$$; + +create function public.publish_package_upgrade( + handle app.valid_name, + package_partial_name app.valid_name, + from_version text, + to_version text, + sql text +) + returns public.package_versions + language plpgsql +as $$ +declare + acc app.accounts = acc from app.accounts acc where id = auth.uid(); + package_id uuid; + package_version_id uuid; +begin + select + ap.id + from + app.packages ap + where + ap.handle = $1 + and ap.partial_name = $2 + into + package_id; + + if package_id is null then + perform app.exception('Unknown package' || handle || '-' || package_partial_name); + end if; + + insert into app.packages(partial_name, handle, description_md) + values (package_partial_name, package_handle) + on conflict do update + set description_md = excluded.description_md + returning id + into package_id; + + -- Insert package_version + insert into app.package_versions(package_id, version_struct, sql) + values ( + package_id, + app.text_to_semver(version), + sql + ) + returning id + into package_version_id; + + -- Return the package version + return pv from public.package_versions pv where pv.id = package_version_id; +end; +$$; + +create function public.is_handle_available(handle app.valid_name) + returns boolean + stable + language sql +as $$ + select + not exists( + select + 1 + from + app.handle_registry hr + where + hr.handle = $1 + ) +$$; +*/ + +create or replace function public.search_packages( + handle citext default null, + partial_name citext default null +) + returns setof public.packages + stable + language sql +as $$ + select * + from public.packages + where + ($1 is null or handle <% $1 or handle ~ $1) + and + ($2 is null or partial_name <% $2 or partial_name ~ $2) + order by + coalesce(extensions.similarity($1, handle), 0) + coalesce(extensions.similarity($2, partial_name), 0) desc, + created_at desc; +$$; diff --git a/crates/pgls_pretty_print/tests/data/multi/20230323180034_reserved_user_accts.sql b/crates/pgls_pretty_print/tests/data/multi/20230323180034_reserved_user_accts.sql new file mode 100644 index 000000000..13f38edec --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230323180034_reserved_user_accts.sql @@ -0,0 +1,82 @@ +insert into auth.users(instance_id, id, aud, role, email, encrypted_password, email_confirmed_at, raw_app_meta_data, raw_user_meta_data, is_sso_user) +values + ( + '00000000-0000-0000-0000-000000000000', gen_random_uuid(), 'authenticated', 'authenticated', 'oliver@oliverrice.com', 'TBD', now(), + '{"provider": "email", "providers": ["email"]}', '{"handle": "olirice", "display_name": "Oli", "bio": "Supabase Staff"}', false + ), + ( + '00000000-0000-0000-0000-000000000000', gen_random_uuid(), 'authenticated', 'authenticated', 'alaister@supabase.io', 'TBD', now(), + '{"provider": "email", "providers": ["email"]}', '{"handle": "alaister", "display_name": "Alaister", "bio": "Supabase Staff"}', false + ), + ( + '00000000-0000-0000-0000-000000000000', gen_random_uuid(), 'authenticated', 'authenticated', 'copple@supabase.io', 'TBD', now(), + '{"provider": "email", "providers": ["email"]}', '{"handle": "kiwicopple", "display_name": "Copple", "bio": "Supabase Staff"}', false + ), + ( + '00000000-0000-0000-0000-000000000000', gen_random_uuid(), 'authenticated', 'authenticated', 'michel@supabase.io', 'TBD', now(), + '{"provider": "email", "providers": ["email"]}', '{"handle": "michelp", "display_name": "Michele", "bio": "Supabase Staff"}', false + ), + ( + '00000000-0000-0000-0000-000000000000', gen_random_uuid(), 'authenticated', 'authenticated', 'mark@supabase.io', 'TBD', now(), + '{"provider": "email", "providers": ["email"]}', '{"handle": "burggraf", "display_name": "Mark", "bio": "Supabase Staff"}', false + ); + +insert into app.handle_registry(handle, is_organization) +values + ('supabase', true), + ('langchain', true), + -- Reserve common impersonation handles + ('admin', false), + ('administrator', false), + ('superuser', false), + ('superadmin', false), + ('root', false), + ('user', false), + ('guest', false), + ('anon', false), + ('authenticated', false), + ('sysadmin', false), + ('support', false), + ('manager', false), + ('default', false), + ('staff', false), + ('help', false), + ('helpdesk', false), + ('test', false), + ('password', false), + ('demo', false), + ('service', false), + ('info', false), + ('webmaster', false), + ('security', false), + ('installer', false); + +begin; + -- Required for trigger on handle registry + select app.simulate_login('oliver@oliverrice.com'); + + insert into app.organizations(handle, display_name, bio) + values + ('supabase', 'Supabase', 'Build in a weekend, scale to millions'); +end; + +insert into app.members(organization_id, account_id, role) +select + o.id, + acc.id, + 'maintainer' +from + app.organizations o, + app.accounts acc +where + -- olirice is already a member because that account created it + acc.handle <> 'olirice'; + +begin; + -- Required for trigger on handle registry + select app.simulate_login('oliver@oliverrice.com'); + + insert into app.organizations(handle, display_name, bio) + values + ('langchain', 'LangChain', 'LangChain is a framework for developing applications powered by language models'); +end; diff --git a/crates/pgls_pretty_print/tests/data/multi/20230328185043_olirice_asciiplot.sql b/crates/pgls_pretty_print/tests/data/multi/20230328185043_olirice_asciiplot.sql new file mode 100644 index 000000000..63ce9929c --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230328185043_olirice_asciiplot.sql @@ -0,0 +1,162 @@ +insert into app.packages( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values ('olirice', 'asciiplot', 'A Toy ASCII Plotting Library', false, '{}'); + +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_name = 'olirice-asciiplot'), +(0,0,1), +$asciiplot$ +CREATE TYPE scatter_state AS ( + x_arr NUMERIC[], + y_arr NUMERIC[], + title TEXT, + height INTEGER, + width INTEGER +); + +CREATE OR REPLACE FUNCTION scatter_sfunc( + state scatter_state, + x numeric, + y numeric, + title TEXT, + height INTEGER, + width INTEGER +) +RETURNS scatter_state +LANGUAGE plpgsql +AS $$ +BEGIN + state.x_arr := array_append(coalesce(state.x_arr, array[]::numeric[]), x); + state.y_arr := array_append(coalesce(state.y_arr, array[]::numeric[]), y); + state.title := coalesce(state.title, title); + state.height := coalesce(state.height, height); + state.width := coalesce(state.width, width); + RETURN state; +END; +$$; + + +CREATE OR REPLACE FUNCTION scatter_internal( + state scatter_state +) +RETURNS TEXT +LANGUAGE plpgsql +AS $$ +DECLARE + plot text[] := array[]::text[]; + + i int := 0; + j int := 0; + max_x numeric := max(v) FROM unnest(state.x_arr) arr(v); + max_y numeric := max(v) FROM unnest(state.y_arr) arr(v); + min_x numeric := min(v) FROM unnest(state.x_arr) arr(v); + min_y numeric := min(v) FROM unnest(state.y_arr) arr(v); + x_range numeric := abs(max_x - min_x); + y_range numeric := abs(max_y - min_y); + + x_scale numeric := x_range / (state.width); + y_scale numeric := y_range / (state.height - 2); + point_idx int; + header text; +begin + for i in 1..(state.height - 2) loop + plot := array_append(plot, repeat(' ', state.width)); + end loop; + + for point_idx in 1..array_length(state.x_arr, 1) loop + i := round((state.y_arr[point_idx] - min_y) / y_scale)::integer; + j := round((state.x_arr[point_idx] - min_x) / x_scale)::integer; + + plot[i] := overlay(plot[i] placing '*' from j for 1); + end loop; + + header = ( + repeat(' ', (state.width - character_length(state.title)) / 2) + || state.title + || repeat(' ', (state.width - character_length(state.title)) / 2) + ); + header = header || E'\n' || repeat('-', state.width) || E'\n'; + + return + header || string_agg(v_elem, E'\n' order by ix desc) + from + unnest(plot) with ordinality v_arr(v_elem, ix); +end; +$$; + + +CREATE AGGREGATE scatter( + x NUMERIC, + y NUMERIC, + title TEXT, + height INTEGER, + width INTEGER +) ( + STYPE = scatter_state, + SFUNC = scatter_sfunc, + FINALFUNC = scatter_internal +); + +comment on type scatter_state is e'internal'; + +$asciiplot$, + +$description$ +# asciiplot + +asciiplot is a toy library for producing ASCII scatterplots from PostgreSQL queries. +Please note that it is not indended for serious use. + +### Usage + +```sql +select + scatter( + val::numeric, --x + val::numeric, -- y + 'stonks!', -- title + 15, -- height + 50 --width + ) +from + generate_series(1,10) vals(val) + +/* + stonks! +---------------------------------------------- +| * +| +| * +| * +| +| * +| +| * +| * +| +| * +| +| * +| * +*/ +``` +$description$ + +); + + +insert into app.package_upgrades(package_id, from_version_struct, to_version_struct, sql) +values ( +(select id from app.packages where package_name = 'olirice-asciiplot'), +(0,0,1), +(0,0,2), +$asciiplot$ +comment on type scatter_state is e'internal'; +$asciiplot$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230330155137_supabase_dbdev.sql b/crates/pgls_pretty_print/tests/data/multi/20230330155137_supabase_dbdev.sql new file mode 100644 index 000000000..18b6d2e25 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230330155137_supabase_dbdev.sql @@ -0,0 +1,284 @@ +insert into app.packages( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values ('supabase', 'dbdev', 'Install pacakges from the dbdev package index', false, '{}'); + + + +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_name = 'supabase-dbdev'), +(0,0,2), +$pkg$ + +create schema dbdev; + +create or replace function dbdev.install(package_name text) + returns bool + language plpgsql +as $$ +declare + -- Endpoint + base_url text = 'https://api.database.dev/rest/v1/'; + apikey text = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyYndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzIsImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJrzM0AQKsu_5k134s'; + + http_ext_schema regnamespace = extnamespace::regnamespace from pg_catalog.pg_extension where extname = 'http' limit 1; + pgtle_is_available bool = true from pg_catalog.pg_extension where extname = 'pg_tle' limit 1; + -- HTTP respones + rec jsonb; + status int; + contents json; + + -- Install Record + rec_sql text; + rec_ver text; + rec_from_ver text; + rec_to_ver text; + rec_package_name text; + rec_description text; + rec_requires text[]; +begin + + if http_ext_schema is null then + raise exception using errcode='22000', message=format('dbdev requires the http extension and it is not available'); + end if; + + if pgtle_is_available is null then + raise exception using errcode='22000', message=format('dbdev requires the pgtle extension and it is not available'); + end if; + + ------------------- + -- Base Versions -- + ------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + 'GET', + format( + '%spackage_versions?select=package_name,version,sql,control_description,control_requires&limit=50&package_name=eq.%s', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + ('apiKey', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> 'status')::int; + contents = to_json(rec ->> 'content') #>> '{}'; + + if status <> 200 then + raise notice using errcode='22000', message=format('DBDEV INFO: %s', contents); + raise exception using errcode='22000', message=format('Non-200 response code while loading versions from dbdev'); + end if; + + if contents is null or json_typeof(contents) <> 'array' or json_array_length(contents) = 0 then + raise exception using errcode='22000', message=format('No versions for package named named %s', package_name); + end if; + + for rec_package_name, rec_ver, rec_sql, rec_description, rec_requires in select + (r ->> 'package_name'), + (r ->> 'version'), + (r ->> 'sql'), + (r ->> 'control_description'), + to_json(rec ->> 'control_requires') #>> '{}' + from + json_array_elements(contents) as r + loop + + if not exists ( + select true + from pgtle.available_extension_versions() + where + -- TLE will not allow multiple full install scripts + -- TODO(OR) open upstream issue to discuss + name = rec_package_name + ) then + perform pgtle.install_extension(rec_package_name, rec_ver, rec_package_name, rec_sql); + end if; + end loop; + + ---------------------- + -- Upgrade Versions -- + ---------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + 'GET', + format( + '%spackage_upgrades?select=package_name,from_version,to_version,sql&limit=50&package_name=eq.%s', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + ('apiKey', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> 'status')::int; + contents = to_json(rec ->> 'content') #>> '{}'; + + if status <> 200 then + raise notice using errcode='22000', message=format('DBDEV INFO: %s', contents); + raise exception using errcode='22000', message=format('Non-200 response code while loading upgrade pathes from dbdev'); + end if; + + if json_typeof(contents) <> 'array' then + raise exception using errcode='22000', message=format('Invalid response from dbdev upgrade pathes'); + end if; + + for rec_package_name, rec_from_ver, rec_to_ver, rec_sql in select + (r ->> 'package_name'), + (r ->> 'from_version'), + (r ->> 'to_version'), + (r ->> 'sql') + from + json_array_elements(contents) as r + loop + + if not exists ( + select true + from pgtle.extension_update_paths(rec_package_name) + where + source = rec_from_ver + and target = rec_to_ver + and path is not null + ) then + perform pgtle.install_update_path(rec_package_name, rec_from_ver, rec_to_ver, rec_sql); + end if; + end loop; + + -------------------------- + -- Send Download Notice -- + -------------------------- + -- Notifies dbdev that a package has been downloaded and records IP + user agent so we can compute unique download counts + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + 'POST', + format( + '%srpc/register_download', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$ + ), + array[ + ('apiKey', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header, + ('x-client-info', 'dbdev/0.0.2')::http_header + ], + 'application/json', + json_build_object('package_name', $stmt$ || pg_catalog.quote_literal($1) || $stmt$)::text + ) + ) x + limit 1; $stmt$ + into rec; + + return true; +end; +$$; + +$pkg$, +$description$ +# dbdev + +dbdev is the SQL client for database.new and is the primary way end users interact with the package (pglet) registry. + +dbdev can be used to load packages from the registry. For example: + +```sql +-- Load the package from the package index +select dbdev.install('olirice-index_advisor'); +``` +Where `olirice` is the handle of the author and `index_advisor` is the name of the pglet. + +Once installed, pglets are visible in PostgreSQL as extensions. At that point they can be enabled with standard Postgres commands i.e. the `create extension` + +To improve reproducibility, we recommend __always__ specifying the package version in your `create extension` statements. + +For example: +```sql +-- Enable the extension +create extension "olirice-index_advisor" + schema 'public' + version '0.1.0'; +``` + +Which creates all tables/indexes/functions/etc specified by the extension. + +## How to Install + +The in-database SQL client for the package registry is named `dbdev`. You can bootstrap the client with: + +```sql +/*--------------------- +---- install dbdev ---- +---------------------- +Requires: + - pg_tle: https://github.com/aws/pg_tle + - pgsql-http: https://github.com/pramsey/pgsql-http +*/ +create extension if not exists http with schema extensions; +create extension if not exists pg_tle; +select pgtle.uninstall_extension_if_exists('supabase-dbdev'); +drop extension if exists "supabase-dbdev"; +select + pgtle.install_extension( + 'supabase-dbdev', + resp.contents ->> 'version', + 'PostgreSQL package manager', + resp.contents ->> 'sql' + ) +from http( + ( + 'GET', + 'https://api.database.dev/rest/v1/' + || 'package_versions?select=sql,version' + || '&package_name=eq.supabase-dbdev' + || '&order=version.desc' + || '&limit=1', + array[ + ( + 'apiKey', + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJp' + || 'c3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyY' + || 'ndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzI' + || 'sImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJ' + || 'rzM0AQKsu_5k134s' + )::http_header + ], + null, + null + ) +) x, +lateral ( + select + ((row_to_json(x) -> 'content') #>> '{}')::json -> 0 +) resp(contents); +create extension "supabase-dbdev"; +select dbdev.install('supabase-dbdev'); +drop extension if exists "supabase-dbdev"; +create extension "supabase-dbdev"; +``` + +With the client ready, search for packages on [database.dev](database.dev) and install them with + +```sql +select dbdev.install('handle-package_name'); +create extension "handle-package_name" + schema 'public' + version '1.2.3'; +``` +$description$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230331145934_burggraf-pg_headerkit.sql b/crates/pgls_pretty_print/tests/data/multi/20230331145934_burggraf-pg_headerkit.sql new file mode 100644 index 000000000..0c91e39fb --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230331145934_burggraf-pg_headerkit.sql @@ -0,0 +1,268 @@ +insert into app.packages( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values ( + 'burggraf', + 'pg_headerkit', + 'PostgreSQL functions that read PostgREST headers for adding functionality to your database', + false, + '{}' +); + + + + +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_name = 'burggraf-pg_headerkit'), +(1,0,0), +$pkg$ +CREATE SCHEMA IF NOT EXISTS hdr; + +CREATE TABLE IF NOT EXISTS hdr.allow_list ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + ip inet NOT NULL, + created_at timestamp with time zone NOT NULL DEFAULT now(), + updated_at timestamp with time zone NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS hdr.deny_list ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + ip inet NOT NULL, + created_at timestamp with time zone NOT NULL DEFAULT now(), + updated_at timestamp with time zone NOT NULL DEFAULT now() +); + +-- get all header values as a json object +CREATE OR REPLACE FUNCTION hdr.headers() RETURNS json + LANGUAGE sql STABLE + AS $$ + SELECT COALESCE(current_setting('request.headers', true)::json, '{}'::json); +$$; + +-- get a header value +CREATE OR REPLACE FUNCTION hdr.header(item text) RETURNS text + LANGUAGE sql STABLE + AS $$ + SELECT COALESCE((current_setting('request.headers', true)::json)->>item, '') +$$; + +-- get the ip address of the current user +CREATE OR REPLACE FUNCTION hdr.ip() RETURNS text + LANGUAGE sql STABLE + AS $$ + SELECT SPLIT_PART(hdr.header('x-forwarded-for') || ',', ',', 1) +$$; + +-- get the allow list +CREATE OR REPLACE FUNCTION hdr.allow_list() RETURNS inet[] + LANGUAGE sql IMMUTABLE + AS $$ + SELECT array_agg(ip) FROM (SELECT ip FROM hdr.allow_list) AS ip; +$$; + +-- get the deny list +CREATE OR REPLACE FUNCTION hdr.deny_list() RETURNS inet[] + LANGUAGE sql IMMUTABLE + AS $$ + SELECT array_agg(ip) FROM (SELECT ip FROM hdr.deny_list) AS ip; +$$; + +-- Is the given ip in the deny list? +CREATE OR REPLACE FUNCTION hdr.in_deny_list(ip inet) RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT + ip = ANY (hdr.deny_list()) +$$; + +-- Is the current user's ip in the deny list? +CREATE OR REPLACE FUNCTION hdr.in_deny_list() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT CASE + WHEN hdr.ip() = '' THEN false + ELSE + (hdr.ip())::inet = ANY (hdr.deny_list()) + END +$$; + +-- Is the given ip in the allow list? +CREATE OR REPLACE FUNCTION hdr.in_allow_list(ip inet) RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT + ip = ANY (hdr.allow_list()) +$$; + +-- Is the current user's ip in the allow list? +CREATE OR REPLACE FUNCTION hdr.in_allow_list() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT CASE + WHEN hdr.ip() = '' THEN false + ELSE + (hdr.ip())::inet = ANY (hdr.allow_list()) + END +$$; + +-- get host, i.e. "localhost:3000" +CREATE OR REPLACE FUNCTION hdr.host() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header('host') +$$; + +-- get origin, i.e. "http://localhost:8100" +CREATE OR REPLACE FUNCTION hdr.origin() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header('origin') +$$; + +-- get referer, i.e. "http://localhost:8100/" +CREATE OR REPLACE FUNCTION hdr.referer() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header('referer') +$$; + +-- get user-agent string +CREATE OR REPLACE FUNCTION hdr.agent() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header('user-agent') +$$; + +-- get x-client-info, i.e. "supabase-js/1.35.7" +CREATE OR REPLACE FUNCTION hdr.client() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header('x-client-info') +$$; + +-- get role (consumer), i.e. "anon-key" +CREATE OR REPLACE FUNCTION hdr.role() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header('x-consumer-username') +$$; + +-- get consumer, i.e. "anon-key" +CREATE OR REPLACE FUNCTION hdr.consumer() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header('x-consumer-username') +$$; + +-- get api server, i.e. "xxxxxxxxxxxxxxxx.supabase.co" +CREATE OR REPLACE FUNCTION hdr.api_host() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header('x-forwarded-host') +$$; + +-- get api server domain, i.e. "xxxxxxxxxxxxxxxx.supabase.co" +CREATE OR REPLACE FUNCTION hdr.domain() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header('x-forwarded-host') +$$; + +-- get project ref #, i.e. "xxxxxxxxxxxxxxxx" +CREATE OR REPLACE FUNCTION hdr.projectref() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header('x-forwarded-host') + SELECT SPLIT_PART(hdr.header('x-forwarded-host') || '.', '.', 1) +$$; + +-- get project ref #, i.e. "xxxxxxxxxxxxxxxx" +CREATE OR REPLACE FUNCTION hdr.ref() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header('x-forwarded-host') + SELECT SPLIT_PART(hdr.header('x-forwarded-host') || '.', '.', 1) +$$; + +-- ********************************************** +-- ********* user-agent parse functions ********* +-- ********************************************** + +-- user-agent parsing for mobile +CREATE OR REPLACE FUNCTION hdr.is_mobile() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header('user-agent') ILIKE '%mobile%' +$$; + +-- user-agent parsing for iPhone +CREATE OR REPLACE FUNCTION hdr.is_iphone() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header('user-agent') ILIKE '%iphone%' +$$; + +-- user-agent parsing for iPad +CREATE OR REPLACE FUNCTION hdr.is_ipad() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header('user-agent') ILIKE '%ipad%' +$$; + +-- user-agent parsing for Android +CREATE OR REPLACE FUNCTION hdr.is_android() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header('user-agent') ILIKE '%android%' +$$; + +$pkg$, + +$description$ +# pg_headerkit: PostgREST Header Kit +A set of functions for adding special features to your application that use PostgREST API calls to your PostgreSQL database. These functions can be used inside PostgreSQL functions that can give your application the following capabilities at the database level: + +- [x] rate limiting +- [x] IP allowlisting +- [x] IP denylisting +- [x] request logging +- [x] request filtering +- [x] request routing +- [x] user allowlisting by uid or email (Supabase-specific) +- [x] user denylisting by uid or email (Supabase-specific) + +### Article +See: [PostgREST Header Hacking](https://github.com/burggraf/postgrest-header-hacking) + +### Function Reference + +| function | description | parameters | returns | +| ------------------------------ | ------------------------------------------------------- | ----------- | ------------------------------ | +| hdr.headers() | get all header values as a json object | none | json object | +| hdr.header(item text) | get a header value | item (text) | text | +| hdr.ip() | get the ip address of the current user | none | text | +| hdr.allow_list() | get the allow list of ip addresses | none | inet[] (array of ip addresses) | +| hdr.deny_list() | get the deny list of ip addresses | none | inet[] (array of ip addresses) | +| hdr.in_deny_list(ip inet) | determine if the given ip is in the deny list | ip (inet) | boolean | +| hdr.in_allow_list(ip inet) | determine if the given ip is in the allow list | ip (inet) | boolean | +| hdr.in_deny_list() | determine if the current user's ip is in the deny list | none | boolean | +| hdr.in_allow_list() | determine if the current user's ip is in the allow list | none | boolean | +| hdr.host() | get host, i.e. "localhost:3000" | none | text | +| hdr.origin() | get origin, i.e. "http://localhost:8100" | none | text | +| hdr.referer() | get referer, i.e. "http://localhost:8100/" | none | text | +| hdr.agent() | get user-agent string | none | text | +| hdr.client() | get x-client-info, i.e. "supabase-js/1.35.7" | none | text | +| hdr.role()
hdr.consumer() | get role (consumer), i.e. "anon-key" | none | text | +| hdr.api_host()
hdr.domain() | get api server, i.e. "xxxxxxxxxxxxxxxx.supabase.co" | none | text | +| hdr.projectref()
hdr.ref() | get project ref #, i.e. "xxxxxxxxxxxxxxxx" | none | text | +| hdr.is_mobile() | is mobile? | none | boolean | +| hdr.is_iphone() | is iphone? | none | boolean | +| hdr.is_ipad() | is ipad? | none | boolean | +| hdr.is_android() | is android? | none | boolean | +$description$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230331163908_olirice-index_advisor.sql b/crates/pgls_pretty_print/tests/data/multi/20230331163908_olirice-index_advisor.sql new file mode 100644 index 000000000..dee44a4b3 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230331163908_olirice-index_advisor.sql @@ -0,0 +1,324 @@ +insert into app.packages( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values ( + 'olirice', + 'index_advisor', + 'Recommend indexes for a given SQL query', + true, + '{hypopg}' +); + + +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_name = 'olirice-index_advisor'), +(0,1,0), +$pkg$ + + +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + hypopg_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = 'hypopg' + and installed_version is not null + ); + begin + + if not hypopg_exists then + raise + exception '"olirice-index_advisor" requires "hypopg"' + using hint = 'Run "create extension hypopg" and try again'; + end if; + end +$$; + + +create type index_advisor_output as ( + index_statements text[], + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb +); + +create function index_advisor( + query text +) + returns table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[] + ) + volatile + language plpgsql + as $$ +declare + n_args int; + prepared_statement_name text = 'index_advisor_working_statement'; + hypopg_schema_name text = (select extnamespace::regnamespace::text from pg_extension where extname = 'hypopg'); + explain_plan_statement text; + rec record; + plan_initial jsonb; + plan_final jsonb; + statements text[] = '{}'; +begin + + -- Disallow multiple statements + if query ilike '%;%' then + raise exception 'query must not contain a semicolon'; + end if; + + -- Hack to support PostgREST because the prepared statement for args incorrectly defaults to text + query := replace(query, 'WITH pgrst_payload AS (SELECT $1 AS json_data)', 'WITH pgrst_payload AS (SELECT $1::json AS json_data)'); + + -- Create a prepared statement for the given query + deallocate all; + execute format('prepare %I as %s', prepared_statement_name, query); + + -- Detect how many arguments are present in the prepared statement + n_args = ( + select + coalesce(array_length(parameter_types, 1), 0) + from + pg_prepared_statements + where + name = prepared_statement_name + limit + 1 + ); + + -- Create a SQL statement that can be executed to collect the explain plan + explain_plan_statement = format( + 'set local plan_cache_mode = force_generic_plan; explain (format json) execute %I%s', + --'explain (format json) execute %I%s', + prepared_statement_name, + case + when n_args = 0 then '' + else format( + '(%s)', array_to_string(array_fill('null'::text, array[n_args]), ',') + ) + end + ); + + -- Store the query plan before any new indexes + execute explain_plan_statement into plan_initial; + + -- Create possible indexes + for rec in ( + with extension_regclass as ( + select + distinct objid as oid + from + pg_depend + where + deptype = 'e' + ) + select + pc.relnamespace::regnamespace::text as schema_name, + pc.relname as table_name, + pa.attname as column_name, + format( + 'select %I.hypopg_create_index($i$create index on %I.%I(%I)$i$)', + hypopg_schema_name, + pc.relnamespace::regnamespace::text, + pc.relname, + pa.attname + ) hypopg_statement + from + pg_catalog.pg_class pc + join pg_catalog.pg_attribute pa + on pc.oid = pa.attrelid + left join extension_regclass er + on pc.oid = er.oid + left join pg_index pi + on pc.oid = pi.indrelid + and (select array_agg(x) from unnest(pi.indkey) v(x)) = array[pa.attnum] + and pi.indexprs is null -- ignore expression indexes + and pi.indpred is null -- ignore partial indexes + where + pc.relnamespace::regnamespace::text not in ( -- ignore schema list + 'pg_catalog', 'pg_toast', 'information_schema' + ) + and er.oid is null -- ignore entities owned by extensions + and pc.relkind in ('r', 'm') -- regular tables, and materialized views + and pc.relpersistence = 'p' -- permanent tables (not unlogged or temporary) + and pa.attnum > 0 + and not pa.attisdropped + and pi.indrelid is null + and pa.atttypid in (20,16,1082,1184,1114,701,23,21,700,1083,2950,1700,25,18,1042,1043) + ) + loop + -- Create the hypothetical index + execute rec.hypopg_statement; + end loop; + + -- Create a prepared statement for the given query + -- The original prepared statement MUST be dropped because its plan is cached + execute format('deallocate %I', prepared_statement_name); + execute format('prepare %I as %s', prepared_statement_name, query); + + -- Store the query plan after new indexes + execute explain_plan_statement into plan_final; + + + -- Idenfity referenced indexes in new plan + execute format( + 'select + coalesce(array_agg(hypopg_get_indexdef(indexrelid) order by indrelid, indkey::text), $i${}$i$::text[]) + from + %I.hypopg() + where + %s ilike ($i$%%$i$ || indexname || $i$%%$i$) + ', + hypopg_schema_name, + quote_literal(plan_final)::text + ) into statements; + + -- Reset all hypothetical indexes + perform hypopg_reset(); + + -- Reset prepared statements + deallocate all; + + return query values ( + (plan_initial -> 0 -> 'Plan' -> 'Startup Cost'), + (plan_final -> 0 -> 'Plan' -> 'Startup Cost'), + (plan_initial -> 0 -> 'Plan' -> 'Total Cost'), + (plan_final -> 0 -> 'Plan' -> 'Total Cost'), + statements::text[] + ); + +end; +$$; + +$pkg$, + +$description_md$ + +# index_advisor + +`index_advisor` is an extension that recommends indexes to improve performance of a given query. + +## Installation + +Note: + +`hypopg` is a dependency of index_advisor. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +```sql +select dbdev.install('olirice-index_advisor'); +create extension if not exists hypopg; +create extension "olirice-index_advisor" cascade; +``` + +## Example + +For a simple example, consider the following table: + +```sql +create table book( + id int primary key, + title text not null +); +``` + +Lets say we want to query `book` by `title`, and return the relevant `id`. +That query would be `select book.id from book where title = $1`. + +We can get `index_advisor` to recommend indexes that would improve performance on that query as follows: + + +```sql + +select + * +from + index_advisor('select book.id from book where title = $1'); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements +---------------------+--------------------+-------------------+------------------+----------------------------------------------------- + 0.00 | 1.17 | 25.88 | 6.40 | {"CREATE INDEX ON public.book USING btree (title)"}, +(1 row) +``` + +where the output columns show top level statistics from the query explain plan (startup_cost, total_cost) and an array of `index_statements` that improve `total_cost`. + +## Features: + +- Generic parameters e.g. `$1`, `$2` +- Support for Materialized Views +- Identifies Tables/Columns Oobfuscaed by Views + +## Usage + +`index_advisor` is not limited to simple use cases. A more complex example could be: + +```sql +select + * +from + index_advisor(' + select + book.id, + book.title, + publisher.name as publisher_name, + author.name as author_name, + review.body review_body + from + book + join publisher + on book.publisher_id = publisher.id + join author + on book.author_id = author.id + join review + on book.id = review.book_id + where + author.id = $1 + and publisher.id = $2 + '); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements +---------------------+--------------------+-------------------+------------------+---------------------------------------------------------- + 27.26 | 12.77 | 68.48 | 42.37 | {"CREATE INDEX ON public.book USING btree (author_id)", + "CREATE INDEX ON public.book USING btree (publisher_id)", + "CREATE INDEX ON public.review USING btree (book_id)"} +(1 row) +``` + +Note: the referenced tables must exist. + +## API + +```sql +index_advisor(query text) +returns + table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[] + ) +``` + +#### Description +For a given *query*, searches for a set of SQL DDL `create index` statements that improve the query's execution time; + +$description_md$ + +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230331163909_olirice-read_once.sql b/crates/pgls_pretty_print/tests/data/multi/20230331163909_olirice-read_once.sql new file mode 100644 index 000000000..6b6fd4fa8 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230331163909_olirice-read_once.sql @@ -0,0 +1,161 @@ +insert into app.packages( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values ( + 'olirice', + 'read_once', + 'Send messages that can only be read once', + false, + '{pg_cron}' +); + + +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_name = 'olirice-read_once'), +(0,3,1), +$pkg$ + +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + pg_cron_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = 'pg_cron' + and installed_version is not null + ); + begin + + if not pg_cron_exists then + raise + exception '"olirice-read_once" requires "pg_cron"' + using hint = 'Run "create extension pg_cron" and try again'; + end if; + end +$$; + + +create schema read_once; + +create unlogged table read_once.messages( + id uuid primary key default gen_random_uuid(), + contents text not null default '', + created_at timestamp default now() +); + +revoke all on read_once.messages from public; +revoke usage on schema read_once from public; + +create or replace function send_message( + contents text +) + returns uuid + security definer + volatile + strict + language sql + as +$$ + insert into read_once.messages(contents) + values ($1) + returning id; +$$; + +create or replace function read_message(id uuid) + returns text + security definer + volatile + strict + language sql + as +$$ + delete from read_once.messages + where read_once.messages.id = $1 + returning contents; +$$ +$pkg$, + +$description_md$ + +# read_once + +A Supabase application for sending messages that can only be read once + +Features: +- messages can only be read one time +- messages are not logged in PostgreSQL write-ahead-log (WAL) + +## Installation + +`pg_cron` is a dependency of `read_once`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + +To expose the `send_message` and `read_message` functions over HTTP, install the extension in a schema that is on the search_path. + + +For example: +```sql +select dbdev.install('olirice-read_once'); +create extension if not exists pg_cron; +create extension "olirice-read_once" + schema public + version '0.3.1'; +``` + + +## HTTP API + +### Create a Message + +```sh +curl -X POST https://.supabase.co/rest/v1/rpc/send_message \ + -H 'apiKey: ' \ + -H 'Content-Type: application/json' + --data-raw '{"contents": "hello, dbdev!"} + +# Returns: "2989156b-2356-4543-9d1b-19dfb8ec3268" +``` + +### Read a Message + +```sh +curl -X https://.supabase.co/rest/v1/rpc/read_message + -H 'apiKey: ' \ + -H 'Content-Type: application/json' \ + --data-raw '{"id": "2989156b-2356-4543-9d1b-19dfb8ec3268"} + +# Returns: "hello, dbdev!" +``` + + +## SQL API + +### Create a Message + +```sql +-- Creates a new messages and returns its unique id +create or replace function send_message( + contents text +) + returns uuid +``` + +### Read a Message + +```sql +-- Read a message by its id +create or replace function read_message( + id uuid +) + returns text +``` +$description_md$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230404162614_michelp-adminpack.sql b/crates/pgls_pretty_print/tests/data/multi/20230404162614_michelp-adminpack.sql new file mode 100644 index 000000000..e22a2a563 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230404162614_michelp-adminpack.sql @@ -0,0 +1,1564 @@ +insert into app.packages( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values ('michelp', 'adminpack', 'A bunch of useful queries for DBA to manage and inspect databases', false, '{}'); + +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_name = 'michelp-adminpack'), +(0,0,1), +$adminpack$ +-- From: https://github.com/ioguix/pgsql-bloat-estimation + +-- Copyright (c) 2015-2019, Jehan-Guillaume (ioguix) de Rorthais +-- All rights reserved. + +-- Redistribution and use in source and binary forms, with or without +-- modification, are permitted provided that the following conditions are met: + +-- * Redistributions of source code must retain the above copyright notice, this +-- list of conditions and the following disclaimer. + +-- * Redistributions in binary form must reproduce the above copyright notice, +-- this list of conditions and the following disclaimer in the documentation +-- and/or other materials provided with the distribution. + +-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +-- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +-- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +-- DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +-- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +-- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +-- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +-- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +-- OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +-- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-- WARNING: executed with a non-superuser role, the query inspect only index on tables you are granted to read. +-- WARNING: rows with is_na = 't' are known to have bad statistics ("name" type is not supported). +-- This query is compatible with PostgreSQL 8.2 and after +CREATE VIEW index_bloat AS SELECT current_database(), nspname AS schemaname, tblname, idxname, bs*(relpages)::bigint AS real_size, + bs*(relpages-est_pages)::bigint AS extra_size, + 100 * (relpages-est_pages)::float / relpages AS extra_pct, + fillfactor, + CASE WHEN relpages > est_pages_ff + THEN bs*(relpages-est_pages_ff) + ELSE 0 + END AS bloat_size, + 100 * (relpages-est_pages_ff)::float / relpages AS bloat_pct, + is_na + -- , 100-(pst).avg_leaf_density AS pst_avg_bloat, est_pages, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples, relpages -- (DEBUG INFO) +FROM ( + SELECT coalesce(1 + + ceil(reltuples/floor((bs-pageopqdata-pagehdr)/(4+nulldatahdrwidth)::float)), 0 -- ItemIdData size + computed avg size of a tuple (nulldatahdrwidth) + ) AS est_pages, + coalesce(1 + + ceil(reltuples/floor((bs-pageopqdata-pagehdr)*fillfactor/(100*(4+nulldatahdrwidth)::float))), 0 + ) AS est_pages_ff, + bs, nspname, tblname, idxname, relpages, fillfactor, is_na + -- , pgstatindex(idxoid) AS pst, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples -- (DEBUG INFO) + FROM ( + SELECT maxalign, bs, nspname, tblname, idxname, reltuples, relpages, idxoid, fillfactor, + ( index_tuple_hdr_bm + + maxalign - CASE -- Add padding to the index tuple header to align on MAXALIGN + WHEN index_tuple_hdr_bm%maxalign = 0 THEN maxalign + ELSE index_tuple_hdr_bm%maxalign + END + + nulldatawidth + maxalign - CASE -- Add padding to the data to align on MAXALIGN + WHEN nulldatawidth = 0 THEN 0 + WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign + ELSE nulldatawidth::integer%maxalign + END + )::numeric AS nulldatahdrwidth, pagehdr, pageopqdata, is_na + -- , index_tuple_hdr_bm, nulldatawidth -- (DEBUG INFO) + FROM ( + SELECT n.nspname, i.tblname, i.idxname, i.reltuples, i.relpages, + i.idxoid, i.fillfactor, current_setting('block_size')::numeric AS bs, + CASE -- MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?) + WHEN version() ~ 'mingw32' OR version() ~ '64-bit|x86_64|ppc64|ia64|amd64' THEN 8 + ELSE 4 + END AS maxalign, + /* per page header, fixed size: 20 for 7.X, 24 for others */ + 24 AS pagehdr, + /* per page btree opaque data */ + 16 AS pageopqdata, + /* per tuple header: add IndexAttributeBitMapData if some cols are null-able */ + CASE WHEN max(coalesce(s.null_frac,0)) = 0 + THEN 8 -- IndexTupleData size + ELSE 8 + (( 32 + 8 - 1 ) / 8) -- IndexTupleData size + IndexAttributeBitMapData size ( max num filed per index + 8 - 1 /8) + END AS index_tuple_hdr_bm, + /* data len: we remove null values save space using it fractionnal part from stats */ + sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) AS nulldatawidth, + max( CASE WHEN i.atttypid = 'pg_catalog.name'::regtype THEN 1 ELSE 0 END ) > 0 AS is_na + FROM ( + SELECT ct.relname AS tblname, ct.relnamespace, ic.idxname, ic.attpos, ic.indkey, ic.indkey[ic.attpos], ic.reltuples, ic.relpages, ic.tbloid, ic.idxoid, ic.fillfactor, + coalesce(a1.attnum, a2.attnum) AS attnum, coalesce(a1.attname, a2.attname) AS attname, coalesce(a1.atttypid, a2.atttypid) AS atttypid, + CASE WHEN a1.attnum IS NULL + THEN ic.idxname + ELSE ct.relname + END AS attrelname + FROM ( + SELECT idxname, reltuples, relpages, tbloid, idxoid, fillfactor, indkey, + pg_catalog.generate_series(1,indnatts) AS attpos + FROM ( + SELECT ci.relname AS idxname, ci.reltuples, ci.relpages, i.indrelid AS tbloid, + i.indexrelid AS idxoid, + coalesce(substring( + array_to_string(ci.reloptions, ' ') + from 'fillfactor=([0-9]+)')::smallint, 90) AS fillfactor, + i.indnatts, + pg_catalog.string_to_array(pg_catalog.textin( + pg_catalog.int2vectorout(i.indkey)),' ')::int[] AS indkey + FROM pg_catalog.pg_index i + JOIN pg_catalog.pg_class ci ON ci.oid = i.indexrelid + WHERE ci.relam=(SELECT oid FROM pg_am WHERE amname = 'btree') + AND ci.relpages > 0 + ) AS idx_data + ) AS ic + JOIN pg_catalog.pg_class ct ON ct.oid = ic.tbloid + LEFT JOIN pg_catalog.pg_attribute a1 ON + ic.indkey[ic.attpos] <> 0 + AND a1.attrelid = ic.tbloid + AND a1.attnum = ic.indkey[ic.attpos] + LEFT JOIN pg_catalog.pg_attribute a2 ON + ic.indkey[ic.attpos] = 0 + AND a2.attrelid = ic.idxoid + AND a2.attnum = ic.attpos + ) i + JOIN pg_catalog.pg_namespace n ON n.oid = i.relnamespace + JOIN pg_catalog.pg_stats s ON s.schemaname = n.nspname + AND s.tablename = i.attrelname + AND s.attname = i.attname + GROUP BY 1,2,3,4,5,6,7,8,9,10,11 + ) AS rows_data_stats + ) AS rows_hdr_pdg_stats +) AS relation_stats +ORDER BY nspname, tblname, idxname; + +CREATE VIEW table_bloat AS /* WARNING: executed with a non-superuser role, the query inspect only tables and materialized view (9.3+) you are granted to read. +* This query is compatible with PostgreSQL 9.0 and more +*/ +SELECT current_database(), schemaname, tblname, bs*tblpages AS real_size, + (tblpages-est_tblpages)*bs AS extra_size, + CASE WHEN tblpages > 0 AND tblpages - est_tblpages > 0 + THEN 100 * (tblpages - est_tblpages)/tblpages::float + ELSE 0 + END AS extra_pct, fillfactor, + CASE WHEN tblpages - est_tblpages_ff > 0 + THEN (tblpages-est_tblpages_ff)*bs + ELSE 0 + END AS bloat_size, + CASE WHEN tblpages > 0 AND tblpages - est_tblpages_ff > 0 + THEN 100 * (tblpages - est_tblpages_ff)/tblpages::float + ELSE 0 + END AS bloat_pct, is_na + -- , tpl_hdr_size, tpl_data_size, (pst).free_percent + (pst).dead_tuple_percent AS real_frag -- (DEBUG INFO) +FROM ( + SELECT ceil( reltuples / ( (bs-page_hdr)/tpl_size ) ) + ceil( toasttuples / 4 ) AS est_tblpages, + ceil( reltuples / ( (bs-page_hdr)*fillfactor/(tpl_size*100) ) ) + ceil( toasttuples / 4 ) AS est_tblpages_ff, + tblpages, fillfactor, bs, tblid, schemaname, tblname, heappages, toastpages, is_na + -- , tpl_hdr_size, tpl_data_size, pgstattuple(tblid) AS pst -- (DEBUG INFO) + FROM ( + SELECT + ( 4 + tpl_hdr_size + tpl_data_size + (2*ma) + - CASE WHEN tpl_hdr_size%ma = 0 THEN ma ELSE tpl_hdr_size%ma END + - CASE WHEN ceil(tpl_data_size)::int%ma = 0 THEN ma ELSE ceil(tpl_data_size)::int%ma END + ) AS tpl_size, bs - page_hdr AS size_per_block, (heappages + toastpages) AS tblpages, heappages, + toastpages, reltuples, toasttuples, bs, page_hdr, tblid, schemaname, tblname, fillfactor, is_na + -- , tpl_hdr_size, tpl_data_size + FROM ( + SELECT + tbl.oid AS tblid, ns.nspname AS schemaname, tbl.relname AS tblname, tbl.reltuples, + tbl.relpages AS heappages, coalesce(toast.relpages, 0) AS toastpages, + coalesce(toast.reltuples, 0) AS toasttuples, + coalesce(substring( + array_to_string(tbl.reloptions, ' ') + FROM 'fillfactor=([0-9]+)')::smallint, 100) AS fillfactor, + current_setting('block_size')::numeric AS bs, + CASE WHEN version()~'mingw32' OR version()~'64-bit|x86_64|ppc64|ia64|amd64' THEN 8 ELSE 4 END AS ma, + 24 AS page_hdr, + 23 + CASE WHEN MAX(coalesce(s.null_frac,0)) > 0 THEN ( 7 + count(s.attname) ) / 8 ELSE 0::int END + + CASE WHEN bool_or(att.attname = 'oid' and att.attnum < 0) THEN 4 ELSE 0 END AS tpl_hdr_size, + sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 0) ) AS tpl_data_size, + bool_or(att.atttypid = 'pg_catalog.name'::regtype) + OR sum(CASE WHEN att.attnum > 0 THEN 1 ELSE 0 END) <> count(s.attname) AS is_na + FROM pg_attribute AS att + JOIN pg_class AS tbl ON att.attrelid = tbl.oid + JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace + LEFT JOIN pg_stats AS s ON s.schemaname=ns.nspname + AND s.tablename = tbl.relname AND s.inherited=false AND s.attname=att.attname + LEFT JOIN pg_class AS toast ON tbl.reltoastrelid = toast.oid + WHERE NOT att.attisdropped + AND tbl.relkind in ('r','m') + GROUP BY 1,2,3,4,5,6,7,8,9,10 + ORDER BY 2,3 + ) AS s + ) AS s2 +) AS s3 +-- WHERE NOT is_na +-- AND tblpages*((pst).free_percent + (pst).dead_tuple_percent)::float4/100 >= 1 +ORDER BY schemaname, tblname; + +-- From https://wiki.postgresql.org/wiki/Lock_dependency_information + +CREATE OR REPLACE VIEW blocking_pid_tree AS +WITH RECURSIVE + lock_composite(requested, current) AS (VALUES + ('AccessShareLock'::text, 'AccessExclusiveLock'::text), + ('RowShareLock'::text, 'ExclusiveLock'::text), + ('RowShareLock'::text, 'AccessExclusiveLock'::text), + ('RowExclusiveLock'::text, 'ShareLock'::text), + ('RowExclusiveLock'::text, 'ShareRowExclusiveLock'::text), + ('RowExclusiveLock'::text, 'ExclusiveLock'::text), + ('RowExclusiveLock'::text, 'AccessExclusiveLock'::text), + ('ShareUpdateExclusiveLock'::text, 'ShareUpdateExclusiveLock'::text), + ('ShareUpdateExclusiveLock'::text, 'ShareLock'::text), + ('ShareUpdateExclusiveLock'::text, 'ShareRowExclusiveLock'::text), + ('ShareUpdateExclusiveLock'::text, 'ExclusiveLock'::text), + ('ShareUpdateExclusiveLock'::text, 'AccessExclusiveLock'::text), + ('ShareLock'::text, 'RowExclusiveLock'::text), + ('ShareLock'::text, 'ShareUpdateExclusiveLock'::text), + ('ShareLock'::text, 'ShareRowExclusiveLock'::text), + ('ShareLock'::text, 'ExclusiveLock'::text), + ('ShareLock'::text, 'AccessExclusiveLock'::text), + ('ShareRowExclusiveLock'::text, 'RowExclusiveLock'::text), + ('ShareRowExclusiveLock'::text, 'ShareUpdateExclusiveLock'::text), + ('ShareRowExclusiveLock'::text, 'ShareLock'::text), + ('ShareRowExclusiveLock'::text, 'ShareRowExclusiveLock'::text), + ('ShareRowExclusiveLock'::text, 'ExclusiveLock'::text), + ('ShareRowExclusiveLock'::text, 'AccessExclusiveLock'::text), + ('ExclusiveLock'::text, 'RowShareLock'::text), + ('ExclusiveLock'::text, 'RowExclusiveLock'::text), + ('ExclusiveLock'::text, 'ShareUpdateExclusiveLock'::text), + ('ExclusiveLock'::text, 'ShareLock'::text), + ('ExclusiveLock'::text, 'ShareRowExclusiveLock'::text), + ('ExclusiveLock'::text, 'ExclusiveLock'::text), + ('ExclusiveLock'::text, 'AccessExclusiveLock'::text), + ('AccessExclusiveLock'::text, 'AccessShareLock'::text), + ('AccessExclusiveLock'::text, 'RowShareLock'::text), + ('AccessExclusiveLock'::text, 'RowExclusiveLock'::text), + ('AccessExclusiveLock'::text, 'ShareUpdateExclusiveLock'::text), + ('AccessExclusiveLock'::text, 'ShareLock'::text), + ('AccessExclusiveLock'::text, 'ShareRowExclusiveLock'::text), + ('AccessExclusiveLock'::text, 'ExclusiveLock'::text), + ('AccessExclusiveLock'::text, 'AccessExclusiveLock'::text) + ) +, lock AS ( + SELECT pid, + virtualtransaction, + granted, + mode, + (locktype, + CASE locktype + WHEN 'relation' THEN concat_ws(';', 'db:'||datname, 'rel:'||relation::regclass::text) + WHEN 'extend' THEN concat_ws(';', 'db:'||datname, 'rel:'||relation::regclass::text) + WHEN 'page' THEN concat_ws(';', 'db:'||datname, 'rel:'||relation::regclass::text, 'page#'||page::text) + WHEN 'tuple' THEN concat_ws(';', 'db:'||datname, 'rel:'||relation::regclass::text, 'page#'||page::text, 'tuple#'||tuple::text) + WHEN 'transactionid' THEN transactionid::text + WHEN 'virtualxid' THEN virtualxid::text + WHEN 'object' THEN concat_ws(';', 'class:'||classid::regclass::text, 'objid:'||objid, 'col#'||objsubid) + ELSE concat('db:'||datname) + END::text) AS target + FROM pg_catalog.pg_locks + LEFT JOIN pg_catalog.pg_database ON (pg_database.oid = pg_locks.database) + ) +, waiting_lock AS ( + SELECT + blocker.pid AS blocker_pid, + blocked.pid AS pid, + concat(blocked.mode,blocked.target) AS lock_target + FROM lock blocker + JOIN lock blocked + ON ( NOT blocked.granted + AND blocker.granted + AND blocked.pid != blocker.pid + AND blocked.target IS NOT DISTINCT FROM blocker.target) + JOIN lock_composite c ON (c.requested = blocked.mode AND c.current = blocker.mode) + ) +, acquired_lock AS ( + WITH waiting AS ( + SELECT lock_target, count(lock_target) AS wait_count FROM waiting_lock GROUP BY lock_target + ) + SELECT + pid, + array_agg(concat(mode,target,' + '||wait_count) ORDER BY wait_count DESC NULLS LAST) AS locks_acquired + FROM lock + LEFT JOIN waiting ON waiting.lock_target = concat(mode,target) + WHERE granted + GROUP BY pid + ) +, blocking_lock AS ( + SELECT + ARRAY[date_part('epoch', query_start)::int, pid] AS seq, + 0::int AS depth, + -1::int AS blocker_pid, + pid, + concat('Connect: ',usename,' ',datname,' ',coalesce(host(client_addr)||':'||client_port, 'local') + , E'\nSQL: ',replace(substr(coalesce(query,'N/A'), 1, 60), E'\n', ' ') + , E'\nAcquired:\n ' + , array_to_string(locks_acquired[1:5] || + CASE WHEN array_upper(locks_acquired,1) > 5 + THEN '... '||(array_upper(locks_acquired,1) - 5)::text||' more ...' + END, + E'\n ') + ) AS lock_info, + concat(to_char(query_start, CASE WHEN age(query_start) > '24h' THEN 'Day DD Mon' ELSE 'HH24:MI:SS' END),E' started\n' + ,CASE WHEN wait_event IS NOT NULL THEN 'waiting' ELSE state END,E'\n' + ,date_trunc('second',age(now(),query_start)),' ago' + ) AS lock_state + FROM acquired_lock blocker + LEFT JOIN pg_stat_activity act USING (pid) + WHERE EXISTS + (SELECT 'x' FROM waiting_lock blocked WHERE blocked.blocker_pid = blocker.pid) + AND NOT EXISTS + (SELECT 'x' FROM waiting_lock blocked WHERE blocked.pid = blocker.pid) +UNION ALL + SELECT + blocker.seq || blocked.pid, + blocker.depth + 1, + blocker.pid, + blocked.pid, + concat('Connect: ',usename,' ',datname,' ',coalesce(host(client_addr)||':'||client_port, 'local') + , E'\nSQL: ',replace(substr(coalesce(query,'N/A'), 1, 60), E'\n', ' ') + , E'\nWaiting: ',blocked.lock_target + , CASE WHEN locks_acquired IS NOT NULL + THEN E'\nAcquired:\n ' || + array_to_string(locks_acquired[1:5] || + CASE WHEN array_upper(locks_acquired,1) > 5 + THEN '... '||(array_upper(locks_acquired,1) - 5)::text||' more ...' + END, + E'\n ') + END + ) AS lock_info, + concat(to_char(query_start, CASE WHEN age(query_start) > '24h' THEN 'Day DD Mon' ELSE 'HH24:MI:SS' END),E' started\n' + ,CASE WHEN wait_event IS NOT NULL THEN 'waiting' ELSE state END,E'\n' + ,date_trunc('second',age(now(),query_start)),' ago' + ) AS lock_state + FROM blocking_lock blocker + JOIN waiting_lock blocked + ON (blocked.blocker_pid = blocker.pid) + LEFT JOIN pg_stat_activity act ON (act.pid = blocked.pid) + LEFT JOIN acquired_lock acq ON (acq.pid = blocked.pid) + WHERE blocker.depth < 5 + ) +SELECT concat(lpad('=> ', 4*depth, ' '),pid::text) AS "PID" +, lock_info AS "Lock Info" +, lock_state AS "State" +FROM blocking_lock +ORDER BY seq; + + +-- From https://wiki.postgresql.org/wiki/Index_Maintenance + +CREATE VIEW duplicate_indexes AS SELECT pg_size_pretty(sum(pg_relation_size(idx))::bigint) as size, + (array_agg(idx))[1] as idx1, (array_agg(idx))[2] as idx2, + (array_agg(idx))[3] as idx3, (array_agg(idx))[4] as idx4 +FROM ( + SELECT indexrelid::regclass as idx, (indrelid::text ||E'\n'|| indclass::text ||E'\n'|| indkey::text ||E'\n'|| + coalesce(indexprs::text,'')||E'\n' || coalesce(indpred::text,'')) as key + FROM pg_index) sub +GROUP BY key HAVING count(*)>1 +ORDER BY sum(pg_relation_size(idx)) DESC; + +-- From https://wiki.postgresql.org/wiki/Disk_Usage + +CREATE VIEW table_sizes AS WITH RECURSIVE pg_inherit(inhrelid, inhparent) AS + (select inhrelid, inhparent + FROM pg_inherits + UNION + SELECT child.inhrelid, parent.inhparent + FROM pg_inherit child, pg_inherits parent + WHERE child.inhparent = parent.inhrelid), +pg_inherit_short AS (SELECT * FROM pg_inherit WHERE inhparent NOT IN (SELECT inhrelid FROM pg_inherit)) +SELECT table_schema + , TABLE_NAME + , row_estimate + , pg_size_pretty(total_bytes) AS total + , pg_size_pretty(index_bytes) AS INDEX + , pg_size_pretty(toast_bytes) AS toast + , pg_size_pretty(table_bytes) AS TABLE + , total_bytes::float8 / sum(total_bytes) OVER () AS total_size_share + FROM ( + SELECT *, total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes + FROM ( + SELECT c.oid + , nspname AS table_schema + , relname AS TABLE_NAME + , SUM(c.reltuples) OVER (partition BY parent) AS row_estimate + , SUM(pg_total_relation_size(c.oid)) OVER (partition BY parent) AS total_bytes + , SUM(pg_indexes_size(c.oid)) OVER (partition BY parent) AS index_bytes + , SUM(pg_total_relation_size(reltoastrelid)) OVER (partition BY parent) AS toast_bytes + , parent + FROM ( + SELECT pg_class.oid + , reltuples + , relname + , relnamespace + , pg_class.reltoastrelid + , COALESCE(inhparent, pg_class.oid) parent + FROM pg_class + LEFT JOIN pg_inherit_short ON inhrelid = oid + WHERE relkind IN ('r', 'p') + ) c + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace + ) a + WHERE oid = parent +) a +ORDER BY total_bytes DESC; + +-- From: https://wiki.postgresql.org/wiki/Index_Maintenance + +CREATE VIEW index_usage AS SELECT + t.schemaname, + t.tablename, + c.reltuples::bigint AS num_rows, + pg_size_pretty(pg_relation_size(c.oid)) AS table_size, + psai.indexrelname AS index_name, + pg_size_pretty(pg_relation_size(i.indexrelid)) AS index_size, + CASE WHEN i.indisunique THEN 'Y' ELSE 'N' END AS "unique", + psai.idx_scan AS number_of_scans, + psai.idx_tup_read AS tuples_read, + psai.idx_tup_fetch AS tuples_fetched +FROM + pg_tables t + LEFT JOIN pg_class c ON t.tablename = c.relname + LEFT JOIN pg_index i ON c.oid = i.indrelid + LEFT JOIN pg_stat_all_indexes psai ON i.indexrelid = psai.indexrelid +WHERE + t.schemaname NOT IN ('pg_catalog', 'information_schema') +ORDER BY 1, 2; + +-- From: https://blog.devgenius.io/top-useful-sql-queries-for-postgresql-35ff3355d265 + +CREATE VIEW database_sizes AS SELECT pg_database.datname, + pg_size_pretty(pg_database_size(pg_database.datname)) AS size +FROM pg_database +ORDER BY pg_database_size(pg_database.datname) DESC; + +CREATE VIEW schema_sizes AS SELECT A.schemaname, + pg_size_pretty (SUM(pg_relation_size(C.oid))) as table, + pg_size_pretty (SUM(pg_total_relation_size(C.oid)-pg_relation_size(C.oid))) as index, + pg_size_pretty (SUM(pg_total_relation_size(C.oid))) as table_index, + SUM(n_live_tup) +FROM pg_class C +LEFT JOIN pg_namespace N ON (N.oid = C .relnamespace) +INNER JOIN pg_stat_user_tables A ON C.relname = A.relname +WHERE nspname NOT IN ('pg_catalog', 'information_schema') +AND C .relkind <> 'i' +AND nspname !~ '^pg_toast' +GROUP BY A.schemaname; + +CREATE VIEW last_vacuum_analyze AS SELECT relname, + last_vacuum, + last_autovacuum + n_mod_since_analyze, + last_analyze, + last_autoanalyze, + analyze_count, + autoanalyze_count + FROM pg_stat_user_tables; + +CREATE VIEW table_row_estimates AS SELECT + schemaname, + relname, + n_live_tup +FROM + pg_stat_user_tables +ORDER BY + n_live_tup DESC; + +-- postgres-meta queries as views from: https://github.com/supabase/postgres-meta + +CREATE VIEW pgmeta_columns AS SELECT + c.oid :: int8 AS table_id, + nc.nspname AS schema, + c.relname AS table, + (c.oid || '.' || a.attnum) AS id, + a.attnum AS ordinal_position, + a.attname AS name, + CASE + WHEN a.atthasdef THEN pg_get_expr(ad.adbin, ad.adrelid) + ELSE NULL + END AS default_value, + CASE + WHEN t.typtype = 'd' THEN CASE + WHEN bt.typelem <> 0 :: oid + AND bt.typlen = -1 THEN 'ARRAY' + WHEN nbt.nspname = 'pg_catalog' THEN format_type(t.typbasetype, NULL) + ELSE 'USER-DEFINED' + END + ELSE CASE + WHEN t.typelem <> 0 :: oid + AND t.typlen = -1 THEN 'ARRAY' + WHEN nt.nspname = 'pg_catalog' THEN format_type(a.atttypid, NULL) + ELSE 'USER-DEFINED' + END + END AS data_type, + COALESCE(bt.typname, t.typname) AS format, + a.attidentity IN ('a', 'd') AS is_identity, + CASE + a.attidentity + WHEN 'a' THEN 'ALWAYS' + WHEN 'd' THEN 'BY DEFAULT' + ELSE NULL + END AS identity_generation, + a.attgenerated IN ('s') AS is_generated, + NOT ( + a.attnotnull + OR t.typtype = 'd' AND t.typnotnull + ) AS is_nullable, + ( + c.relkind IN ('r', 'p') + OR c.relkind IN ('v', 'f') AND pg_column_is_updatable(c.oid, a.attnum, FALSE) + ) AS is_updatable, + uniques.table_id IS NOT NULL AS is_unique, + array_to_json( + array( + SELECT + enumlabel + FROM + pg_catalog.pg_enum enums + WHERE + enums.enumtypid = coalesce(bt.oid, t.oid) + OR enums.enumtypid = coalesce(bt.typelem, t.typelem) + ORDER BY + enums.enumsortorder + ) + ) AS enums, + col_description(c.oid, a.attnum) AS comment +FROM + pg_attribute a + LEFT JOIN pg_attrdef ad ON a.attrelid = ad.adrelid + AND a.attnum = ad.adnum + JOIN ( + pg_class c + JOIN pg_namespace nc ON c.relnamespace = nc.oid + ) ON a.attrelid = c.oid + JOIN ( + pg_type t + JOIN pg_namespace nt ON t.typnamespace = nt.oid + ) ON a.atttypid = t.oid + LEFT JOIN ( + pg_type bt + JOIN pg_namespace nbt ON bt.typnamespace = nbt.oid + ) ON t.typtype = 'd' + AND t.typbasetype = bt.oid + LEFT JOIN ( + SELECT + conrelid AS table_id, + conkey[1] AS ordinal_position + FROM pg_catalog.pg_constraint + WHERE contype = 'u' AND cardinality(conkey) = 1 + ) AS uniques ON uniques.table_id = c.oid AND uniques.ordinal_position = a.attnum +WHERE + NOT pg_is_other_temp_schema(nc.oid) + AND a.attnum > 0 + AND NOT a.attisdropped + AND (c.relkind IN ('r', 'v', 'm', 'f', 'p')) + AND ( + pg_has_role(c.relowner, 'USAGE') + OR has_column_privilege( + c.oid, + a.attnum, + 'SELECT, INSERT, UPDATE, REFERENCES' + ) + ); + +CREATE VIEW pgmeta_config AS SELECT + name, + setting, + category, + TRIM(split_part(category, '/', 1)) AS group, + TRIM(split_part(category, '/', 2)) AS subgroup, + unit, + short_desc, + extra_desc, + context, + vartype, + source, + min_val, + max_val, + enumvals, + boot_val, + reset_val, + sourcefile, + sourceline, + pending_restart +FROM + pg_settings +ORDER BY + category, + name; + +CREATE VIEW pgmeta_extensions AS SELECT + e.name, + n.nspname AS schema, + e.default_version, + x.extversion AS installed_version, + e.comment +FROM + pg_available_extensions() e(name, default_version, comment) + LEFT JOIN pg_extension x ON e.name = x.extname + LEFT JOIN pg_namespace n ON x.extnamespace = n.oid; + +CREATE VIEW pgmeta_foreign_tables AS SELECT + c.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS name, + obj_description(c.oid) AS comment +FROM + pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE + c.relkind = 'f'; + +CREATE VIEW pgmeta_functions AS with functions as ( + select + *, + -- proargmodes is null when all arg modes are IN + coalesce( + p.proargmodes, + array_fill('i'::text, array[cardinality(coalesce(p.proallargtypes, p.proargtypes))]) + ) as arg_modes, + -- proargnames is null when all args are unnamed + coalesce( + p.proargnames, + array_fill(''::text, array[cardinality(coalesce(p.proallargtypes, p.proargtypes))]) + ) as arg_names, + -- proallargtypes is null when all arg modes are IN + coalesce(p.proallargtypes, p.proargtypes) as arg_types, + array_cat( + array_fill(false, array[pronargs - pronargdefaults]), + array_fill(true, array[pronargdefaults])) as arg_has_defaults + from + pg_proc as p + where + p.prokind = 'f' +) +select + f.oid::int8 as id, + n.nspname as schema, + f.proname as name, + l.lanname as language, + case + when l.lanname = 'internal' then '' + else f.prosrc + end as definition, + case + when l.lanname = 'internal' then f.prosrc + else pg_get_functiondef(f.oid) + end as complete_statement, + coalesce(f_args.args, '[]') as args, + pg_get_function_arguments(f.oid) as argument_types, + pg_get_function_identity_arguments(f.oid) as identity_argument_types, + f.prorettype::int8 as return_type_id, + pg_get_function_result(f.oid) as return_type, + nullif(rt.typrelid::int8, 0) as return_type_relation_id, + f.proretset as is_set_returning_function, + case + when f.provolatile = 'i' then 'IMMUTABLE' + when f.provolatile = 's' then 'STABLE' + when f.provolatile = 'v' then 'VOLATILE' + end as behavior, + f.prosecdef as security_definer, + f_config.config_params as config_params +from + functions f + left join pg_namespace n on f.pronamespace = n.oid + left join pg_language l on f.prolang = l.oid + left join pg_type rt on rt.oid = f.prorettype + left join ( + select + oid, + jsonb_object_agg(param, value) filter (where param is not null) as config_params + from + ( + select + oid, + (string_to_array(unnest(proconfig), '='))[1] as param, + (string_to_array(unnest(proconfig), '='))[2] as value + from + functions + ) as t + group by + oid + ) f_config on f_config.oid = f.oid + left join ( + select + oid, + jsonb_agg(jsonb_build_object( + 'mode', t2.mode, + 'name', name, + 'type_id', type_id, + 'has_default', has_default + )) as args + from + ( + select + oid, + unnest(arg_modes) as mode, + unnest(arg_names) as name, + unnest(arg_types)::int8 as type_id, + unnest(arg_has_defaults) as has_default + from + functions + ) as t1, + lateral ( + select + case + when t1.mode = 'i' then 'in' + when t1.mode = 'o' then 'out' + when t1.mode = 'b' then 'inout' + when t1.mode = 'v' then 'variadic' + else 'table' + end as mode + ) as t2 + group by + t1.oid + ) f_args on f_args.oid = f.oid; + +CREATE VIEW pgmeta_materialized_views AS select + c.oid::int8 as id, + n.nspname as schema, + c.relname as name, + c.relispopulated as is_populated, + obj_description(c.oid) as comment +from + pg_class c + join pg_namespace n on n.oid = c.relnamespace +where + c.relkind = 'm'; + +CREATE VIEW pgmeta_policies AS SELECT + pol.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS table, + c.oid :: int8 AS table_id, + pol.polname AS name, + CASE + WHEN pol.polpermissive THEN 'PERMISSIVE' :: text + ELSE 'RESTRICTIVE' :: text + END AS action, + CASE + WHEN pol.polroles = '{0}' :: oid [] THEN array_to_json( + string_to_array('public' :: text, '' :: text) :: name [] + ) + ELSE array_to_json( + ARRAY( + SELECT + pg_roles.rolname + FROM + pg_roles + WHERE + pg_roles.oid = ANY (pol.polroles) + ORDER BY + pg_roles.rolname + ) + ) + END AS roles, + CASE + pol.polcmd + WHEN 'r' :: "char" THEN 'SELECT' :: text + WHEN 'a' :: "char" THEN 'INSERT' :: text + WHEN 'w' :: "char" THEN 'UPDATE' :: text + WHEN 'd' :: "char" THEN 'DELETE' :: text + WHEN '*' :: "char" THEN 'ALL' :: text + ELSE NULL :: text + END AS command, + pg_get_expr(pol.polqual, pol.polrelid) AS definition, + pg_get_expr(pol.polwithcheck, pol.polrelid) AS check +FROM + pg_policy pol + JOIN pg_class c ON c.oid = pol.polrelid + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace; + +CREATE VIEW pgmeta_primary_keys AS SELECT + n.nspname AS schema, + c.relname AS table_name, + a.attname AS name, + c.oid :: int8 AS table_id +FROM + pg_index i, + pg_class c, + pg_attribute a, + pg_namespace n +WHERE + i.indrelid = c.oid + AND c.relnamespace = n.oid + AND a.attrelid = c.oid + AND a.attnum = ANY (i.indkey) + AND i.indisprimary; + +CREATE VIEW pgmeta_publications AS SELECT + p.oid :: int8 AS id, + p.pubname AS name, + p.pubowner::regrole::text AS owner, + p.pubinsert AS publish_insert, + p.pubupdate AS publish_update, + p.pubdelete AS publish_delete, + p.pubtruncate AS publish_truncate, + CASE + WHEN p.puballtables THEN NULL + ELSE pr.tables + END AS tables +FROM + pg_catalog.pg_publication AS p + LEFT JOIN LATERAL ( + SELECT + COALESCE( + array_agg( + json_build_object( + 'id', + c.oid :: int8, + 'name', + c.relname, + 'schema', + nc.nspname + ) + ), + '{}' + ) AS tables + FROM + pg_catalog.pg_publication_rel AS pr + JOIN pg_class AS c ON pr.prrelid = c.oid + join pg_namespace as nc on c.relnamespace = nc.oid + WHERE + pr.prpubid = p.oid + ) AS pr ON 1 = 1; + +CREATE VIEW pgmeta_relationships AS SELECT + c.oid :: int8 AS id, + c.conname AS constraint_name, + nsa.nspname AS source_schema, + csa.relname AS source_table_name, + sa.attname AS source_column_name, + nta.nspname AS target_table_schema, + cta.relname AS target_table_name, + ta.attname AS target_column_name +FROM + pg_constraint c + JOIN ( + pg_attribute sa + JOIN pg_class csa ON sa.attrelid = csa.oid + JOIN pg_namespace nsa ON csa.relnamespace = nsa.oid + ) ON sa.attrelid = c.conrelid + AND sa.attnum = ANY (c.conkey) + JOIN ( + pg_attribute ta + JOIN pg_class cta ON ta.attrelid = cta.oid + JOIN pg_namespace nta ON cta.relnamespace = nta.oid + ) ON ta.attrelid = c.confrelid + AND ta.attnum = ANY (c.confkey) +WHERE + c.contype = 'f'; + +CREATE VIEW pgmeta_roles AS -- TODO: Consider using pg_authid vs. pg_roles for unencrypted password field +SELECT + oid :: int8 AS id, + rolname AS name, + rolsuper AS is_superuser, + rolcreatedb AS can_create_db, + rolcreaterole AS can_create_role, + rolinherit AS inherit_role, + rolcanlogin AS can_login, + rolreplication AS is_replication_role, + rolbypassrls AS can_bypass_rls, + ( + SELECT + COUNT(*) + FROM + pg_stat_activity + WHERE + pg_roles.rolname = pg_stat_activity.usename + ) AS active_connections, + CASE WHEN rolconnlimit = -1 THEN current_setting('max_connections') :: int8 + ELSE rolconnlimit + END AS connection_limit, + rolpassword AS password, + rolvaliduntil AS valid_until, + rolconfig AS config +FROM + pg_roles; + +CREATE VIEW pgmeta_schemas AS select + n.oid::int8 as id, + n.nspname as name, + u.rolname as owner +from + pg_namespace n, + pg_roles u +where + n.nspowner = u.oid + and ( + pg_has_role(n.nspowner, 'USAGE') + or has_schema_privilege(n.oid, 'CREATE, USAGE') + ) + and not pg_catalog.starts_with(n.nspname, 'pg_temp_') + and not pg_catalog.starts_with(n.nspname, 'pg_toast_temp_'); + +CREATE VIEW pgmeta_tables AS SELECT + c.oid :: int8 AS id, + nc.nspname AS schema, + c.relname AS name, + c.relrowsecurity AS rls_enabled, + c.relforcerowsecurity AS rls_forced, + CASE + WHEN c.relreplident = 'd' THEN 'DEFAULT' + WHEN c.relreplident = 'i' THEN 'INDEX' + WHEN c.relreplident = 'f' THEN 'FULL' + ELSE 'NOTHING' + END AS replica_identity, + pg_total_relation_size(format('%I.%I', nc.nspname, c.relname)) :: int8 AS bytes, + pg_size_pretty( + pg_total_relation_size(format('%I.%I', nc.nspname, c.relname)) + ) AS size, + pg_stat_get_live_tuples(c.oid) AS live_rows_estimate, + pg_stat_get_dead_tuples(c.oid) AS dead_rows_estimate, + obj_description(c.oid) AS comment +FROM + pg_namespace nc + JOIN pg_class c ON nc.oid = c.relnamespace +WHERE + c.relkind IN ('r', 'p') + AND NOT pg_is_other_temp_schema(nc.oid) + AND ( + pg_has_role(c.relowner, 'USAGE') + OR has_table_privilege( + c.oid, + 'SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER' + ) + OR has_any_column_privilege(c.oid, 'SELECT, INSERT, UPDATE, REFERENCES') + ); + + +CREATE VIEW pgmeta_triggers AS SELECT + pg_t.oid AS id, + pg_t.tgrelid AS table_id, + CASE + WHEN pg_t.tgenabled = 'D' THEN 'DISABLED' + WHEN pg_t.tgenabled = 'O' THEN 'ORIGIN' + WHEN pg_t.tgenabled = 'R' THEN 'REPLICA' + WHEN pg_t.tgenabled = 'A' THEN 'ALWAYS' + END AS enabled_mode, + ( + STRING_TO_ARRAY( + ENCODE(pg_t.tgargs, 'escape'), '\000' + ) + )[:pg_t.tgnargs] AS function_args, + is_t.trigger_name AS name, + is_t.event_object_table AS table, + is_t.event_object_schema AS schema, + is_t.action_condition AS condition, + is_t.action_orientation AS orientation, + is_t.action_timing AS activation, + ARRAY_AGG(is_t.event_manipulation)::text[] AS events, + pg_p.proname AS function_name, + pg_n.nspname AS function_schema +FROM + pg_trigger AS pg_t +JOIN + pg_class AS pg_c +ON pg_t.tgrelid = pg_c.oid +JOIN information_schema.triggers AS is_t +ON is_t.trigger_name = pg_t.tgname +AND pg_c.relname = is_t.event_object_table +JOIN pg_proc AS pg_p +ON pg_t.tgfoid = pg_p.oid +JOIN pg_namespace AS pg_n +ON pg_p.pronamespace = pg_n.oid +GROUP BY + pg_t.oid, + pg_t.tgrelid, + pg_t.tgenabled, + pg_t.tgargs, + pg_t.tgnargs, + is_t.trigger_name, + is_t.event_object_table, + is_t.event_object_schema, + is_t.action_condition, + is_t.action_orientation, + is_t.action_timing, + pg_p.proname, + pg_n.nspname; + + +CREATE VIEW pgmeta_types AS select + t.oid::int8 as id, + t.typname as name, + n.nspname as schema, + format_type (t.oid, null) as format, + coalesce(t_enums.enums, '[]') as enums, + coalesce(t_attributes.attributes, '[]') as attributes, + obj_description (t.oid, 'pg_type') as comment +from + pg_type t + left join pg_namespace n on n.oid = t.typnamespace + left join ( + select + enumtypid, + jsonb_agg(enumlabel order by enumsortorder) as enums + from + pg_enum + group by + enumtypid + ) as t_enums on t_enums.enumtypid = t.oid + left join ( + select + oid, + jsonb_agg( + jsonb_build_object('name', a.attname, 'type_id', a.atttypid::int8) + order by a.attnum asc + ) as attributes + from + pg_class c + join pg_attribute a on a.attrelid = c.oid + where + c.relkind = 'c' and not a.attisdropped + group by + c.oid + ) as t_attributes on t_attributes.oid = t.typrelid +where + ( + t.typrelid = 0 + or ( + select + c.relkind = 'c' + from + pg_class c + where + c.oid = t.typrelid + ) + ); + +CREATE VIEW pgmeta_version AS SELECT + version(), + current_setting('server_version_num') :: int8 AS version_number, + ( + SELECT + COUNT(*) AS active_connections + FROM + pg_stat_activity + ) AS active_connections, + current_setting('max_connections') :: int8 AS max_connections; + +CREATE VIEW pgmeta_views AS SELECT + c.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS name, + -- See definition of information_schema.views + (pg_relation_is_updatable(c.oid, false) & 20) = 20 AS is_updatable, + obj_description(c.oid) AS comment +FROM + pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE + c.relkind = 'v'; +$adminpack$, + +$description_md$ +# michelp-adminpack + +A Trusted Language Extension containing a variety of useful databse admin queries. + +Postgres database admins are often faced with a variety of issues that +require them introspecting the database state. This extension +contains a mixed-bag of views that reflect useful information for +admins. + +## Installation + +Using dbdev: + +``` +postgres=> select dbdev.install('michelp-adminpack'); + install +--------- + t +(1 row) + +postgres=> create schema adminpack; +CREATE SCHEMA +postgres=> create extension "michelp-adminpack" with schema adminpack; +CREATE EXTENSION +``` + +This will the extension into the `adminpack` schema, substitute with +some other schema if you want it to go somewhere else. + +## Index Bloat + +Index updates and querying can end up getting slower and slower over +time due to what's called "index bloat". This is where frequent +updates and deletes can cause space in index data structures to go +unused. Understanding when this is happening is not obvious, so there +is a rather complex query you can run to detect it. + +`index_bloat` + +| Column | Type | +|------------------|------------------| +| current_database | name | +| schemaname | name | +| tblname | name | +| idxname | name | +| real_size | numeric | +| extra_size | numeric | +| extra_pct | double precision | +| fillfactor | integer | +| bloat_size | double precision | +| bloat_pct | double precision | +| is_na | boolean | + + +## Table Bloat + +Like index bloat, tables with high update and delete rates can also +end up containing lots of allocated but unused space. This query +shows which tables have the most bloat. + +`table_bloat` + +| Column | Type | +|------------------|------------------| +| current_database | name | +| schemaname | name | +| tblname | name | +| real_size | numeric | +| extra_size | double precision | +| extra_pct | double precision | +| fillfactor | integer | +| bloat_size | double precision | +| bloat_pct | double precision | +| is_na | boolean | + + +## Blocking PID Tree + +Postgres queries can block each other, and this blocking relationship +can form a tree, where A blocks B, which blocks C, and so on. This +query formats blocking queries into a tree structure so it's easy to +see what query is causing the blockage. + +`blocking_pid_tree` + +| Column | Type | +|-----------|------| +| PID | text | +| Lock Info | text | +| State | text | + + +## Duplicate Indexes + +If a table contains duplicate indexes, then unecessary work is done +updating and storing them, this query will show up to 4 duplicate +indexes per table if they exist. + +`duplicate_indexes` + +| Column | Type | +|--------|----------| +| size | text | +| idx1 | regclass | +| idx2 | regclass | +| idx3 | regclass | +| idx4 | regclass | + + +## Table Sizes + +This view shows tables and their sizes. + +`table_sizes` + +| Column | Type | +|------------------|------------------| +| table_schema | name | +| table_name | name | +| row_estimate | real | +| total | text | +| index | text | +| toast | text | +| table | text | +| total_size_share | double precision | + + +## Schema Sizes + +This view shows schemas and their sizes, which is the sum of the sizes +of all the tables and indexes in the schema. + +`schema_sizes` + +| Column | Type | +|-------------|---------| +| schemaname | name | +| table | text | +| index | text | +| table_index | text | +| sum | numeric | + + +## Index Usage + +This view shows index size and usage. An unused index still needs to +be updated and that takes time an storage, so it's a good candidate to +drop. + +`index_usage` + +| Column | Type | +|-----------------|--------| +| schemaname | name | +| tablename | name | +| num_rows | bigint | +| table_size | text | +| index_name | name | +| index_size | text | +| unique | text | +| number_of_scans | bigint | +| tuples_read | bigint | +| tuples_fetched | bigint | + + +## Last Vacuum Analyze + +This views shows the last time a table was vacuumed an analyzed. + +`last_vacuum_analyze` + +| Column | Type | +|---------------------|--------------------------| +| relname | name | +| last_vacuum | timestamp with time zone | +| n_mod_since_analyze | timestamp with time zone | +| last_analyze | timestamp with time zone | +| last_autoanalyze | timestamp with time zone | +| analyze_count | bigint | +| autoanalyze_count | bigint | + +## Table Row Estimates + +This view shows estimates for the number of rows in a table. This is +just an estimate and depends on up to date table statistics. + +`table_row_estimates` + +| Column | Type | +|------------|--------| +| schemaname | name | +| relname | name | +| n_live_tup | bigint | + +## PGMeta Columns + +This view shows infromation for the columns of tables in the system. + +`pgmeta_columns` + +| Column | Type | +|---------------------|----------| +| table_id | bigint | +| schema | name | +| table | name | +| id | text | +| ordinal_position | smallint | +| name | name | +| default_value | text | +| data_type | text | +| format | name | +| is_identity | boolean | +| identity_generation | text | +| is_generated | boolean | +| is_nullable | boolean | +| is_updatable | boolean | +| is_unique | boolean | +| enums | json | +| comment | text | + +## PGMeta Config + +This views shows the configuration of the database. + +`pgmeta_config` + +| Column | Type | +|-----------------|---------| +| name | text | +| setting | text | +| category | text | +| group | text | +| subgroup | text | +| unit | text | +| short_desc | text | +| extra_desc | text | +| context | text | +| vartype | text | +| source | text | +| min_val | text | +| max_val | text | +| enumvals | text[] | +| boot_val | text | +| reset_val | text | +| sourcefile | text | +| sourceline | integer | +| pending_restart | boolean | + +## PGMeta Extensions + +This view shows installed extensions in the database. + +`pgmeta_extensions` + +| Column | Type | +|-------------------|------| +| name | name | +| schema | name | +| default_version | text | +| installed_version | text | +| comment | text | + +## PGMeta Foreign Tables + +This view shows foreign tables in the database. + +`pgmeta_foreign_tables` + +| Column | Type | +|---------|--------| +| id | bigint | +| schema | name | +| name | name | +| comment | text | + +## PGMeta Functions + +This view shows functions in the database. + +`pgmeta_functions` + +| Column | Type | +|---------------------------|---------| +| id | bigint | +| schema | name | +| name | name | +| language | name | +| definition | text | +| complete_statement | text | +| args | jsonb | +| argument_types | text | +| identity_argument_types | text | +| return_type_id | bigint | +| return_type | text | +| return_type_relation_id | bigint | +| is_set_returning_function | boolean | +| behavior | text | +| security_definer | boolean | +| config_params | jsonb | + +## PGMeta Materialized Views + +This view shows materialized views in the database. + +`pgmeta_materialized_views` + +| Column | Type | +|--------------|---------| +| id | bigint | +| schema | name | +| name | name | +| is_populated | boolean | +| comment | text | + +## PGMeta Policies + +This view shows Row Level Security Policies in the database. + +`pgmeta_policies` + +| Column | Type | +|------------|--------| +| id | bigint | +| schema | name | +| table | name | +| table_id | bigint | +| name | name | +| action | text | +| roles | json | +| command | text | +| definition | text | +| check | text | + +## PGMeta Primary Keys + +This view shows primary keys in the database. + +`pgmeta_primary_keys` + +| Column | Type | +|------------|--------| +| schema | name | +| table_name | name | +| name | name | +| table_id | bigint | + +## PGMeta Publications + +This view shows logical replication publishers in the database. + +`pgmeta_publications` + +| Column | Type | +|------------------|---------| +| id | bigint | +| name | name | +| owner | text | +| publish_insert | boolean | +| publish_update | boolean | +| publish_delete | boolean | +| publish_truncate | boolean | +| tables | json[] | + +## PGMeta Relationships + +This view shows foreign key relationships in the database. + +`pgmeta_relationships` + +| Column | Type | +|---------------------|--------| +| id | bigint | +| constraint_name | name | +| source_schema | name | +| source_table_name | name | +| source_column_name | name | +| target_table_schema | name | +| target_table_name | name | +| target_column_name | name | + +## PGMeta Roles + +This view shows roles in the database system. Note that roles are +global objects and apply to all databases. + +`pgmeta_roles` + +| Column | Type | +|---------------------|--------------------------| +| id | bigint | +| name | name | +| is_superuser | boolean | +| can_create_db | boolean | +| can_create_role | boolean | +| inherit_role | boolean | +| can_login | boolean | +| is_replication_role | boolean | +| can_bypass_rls | boolean | +| active_connections | bigint | +| connection_limit | bigint | +| password | text | +| valid_until | timestamp with time zone | +| config | text[] | + +## PGMeta Schemas + +This view shows all schemas in the database. + +`pgmeta_schemas` + +| Column | Type | +|--------|--------| +| id | bigint | +| name | name | +| owner | name | + +## PGMeta Tables + +This view shows all tables in the database. + +`pgmeta_tables` + +| Column | Type | +|--------------------|---------| +| id | bigint | +| schema | name | +| name | name | +| rls_enabled | boolean | +| rls_forced | boolean | +| replica_identity | text | +| bytes | bigint | +| size | text | +| live_rows_estimate | bigint | +| dead_rows_estimate | bigint | +| comment | text | + +## PGMeta Triggers + +This view shows all triggers in the database. + +`pgmeta_triggers` + +| Column | Type | +|-----------------|-----------------------------------| +| id | oid | +| table_id | oid | +| enabled_mode | text | +| function_args | text[] | +| name | information_schema.sql_identifier | +| table | information_schema.sql_identifier | +| schema | information_schema.sql_identifier | +| condition | information_schema.character_data | +| orientation | information_schema.character_data | +| activation | information_schema.character_data | +| events | text[] | +| function_name | name | +| function_schema | name | + +## PGMeta Types + +This view shows all types in the database. + +`pgmeta_types` + +| Column | Type | +|------------|--------| +| id | bigint | +| name | name | +| schema | name | +| format | text | +| enums | jsonb | +| attributes | jsonb | +| comment | text | + +## PGMeta Version + +This view shows the current database version. + +`pgmeta_version` + +| Column | Type | +|--------------------|--------| +| version | text | +| version_number | bigint | +| active_connections | bigint | +| max_connections | bigint | + +## PGMeta Views + +This view shows all views in the database. + +`pgmeta_views` + +| Column | Type | +|--------------|---------| +| id | bigint | +| schema | name | +| name | name | +| is_updatable | boolean | +| comment | text | +$description_md$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230405083103_fix_auth_schema_values.sql b/crates/pgls_pretty_print/tests/data/multi/20230405083103_fix_auth_schema_values.sql new file mode 100644 index 000000000..b98039c95 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230405083103_fix_auth_schema_values.sql @@ -0,0 +1,30 @@ +update auth.users +set + created_at = now(), + updated_at = now(), + email_confirmed_at = now(), + confirmation_token = '', + recovery_token = '', + email_change_token_new = '', + email_change = ''; + +insert into + auth.identities ( + id, + provider_id, + provider, + user_id, + identity_data, + created_at, + updated_at + ) +select + gen_random_uuid(), + email, + 'email' as provider, + id as user_id, + jsonb_build_object('sub', id, 'email', email) as identity_data, + created_at, + updated_at +from + auth.users; diff --git a/crates/pgls_pretty_print/tests/data/multi/20230405085810_fix_avatars_handle.sql b/crates/pgls_pretty_print/tests/data/multi/20230405085810_fix_avatars_handle.sql new file mode 100644 index 000000000..21bfb5c8c --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230405085810_fix_avatars_handle.sql @@ -0,0 +1,41 @@ +create or replace function app.update_avatar_id() + returns trigger + language plpgsql + security definer + as $$ + declare + v_handle app.valid_name; + v_affected_account app.accounts := null; + begin + select (string_to_array(new.name, '/'::text))[1]::app.valid_name into v_handle; + + update app.accounts + set avatar_id = new.id + where handle = v_handle + returning * into v_affected_account; + + if not v_affected_account is null then + update auth.users u + set + "raw_user_meta_data" = u.raw_user_meta_data || jsonb_build_object( + 'avatar_path', new.name + ) + where u.id = v_affected_account.id; + else + update app.organizations + set avatar_id = new.id + where handle = v_handle; + end if; + + return new; + end; + $$; + +alter policy storage_objects_insert_policy + on storage.objects + with check ( + app.is_handle_maintainer( + auth.uid(), + (string_to_array(name, '/'::text))[1]::app.valid_name + ) + ); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230405163940_download_metrics.sql b/crates/pgls_pretty_print/tests/data/multi/20230405163940_download_metrics.sql new file mode 100644 index 000000000..d14c54e20 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230405163940_download_metrics.sql @@ -0,0 +1,93 @@ +-- Return API request headers +create or replace function app.api_request_headers() + returns json + language sql + stable + as +$$ + select coalesce(current_setting('request.headers', true)::json, '{}'::json); +$$; + + +-- Return specific API request header +create or replace function app.api_request_header(text) + returns text + language sql + stable + as +$$ + select app.api_request_headers() ->> $1 +$$; + + +-- IP address of current API request +create or replace function app.api_request_ip() + returns inet + language sql + stable + as +$$ + select split_part(app.api_request_header('x-forwarded-for') || ',', ',', 1)::inet +$$; + +-- IP address of current API request +create or replace function app.api_request_client_info() + returns text + language sql + stable + as +$$ + select app.api_request_header('x-client-info') +$$; + + +create table app.downloads( + id uuid primary key default gen_random_uuid(), + package_id uuid not null references app.packages(id), + ip inet not null default app.api_request_ip()::inet, + client_info text default app.api_request_client_info(), + created_at timestamptz not null default now() +); + +-- Speed up metrics query +create index downloads_package_id_ip + on app.downloads (package_id); + +create index downloads_created_at + on app.downloads + using brin(created_at); + +create or replace function public.register_download(package_name text) + returns void + language sql + security definer + as +$$ + insert into app.downloads(package_id) + select id + from app.packages ap + where ap.package_name = $1 +$$; + + +-- Public facing download metrics view. For website only. Not a stable part of dbdev API +create materialized view public.download_metrics +as + select + dl.package_id, + count(dl.id) downloads_all_time, + count(dl.id) filter (where dl.created_at > now() - '180 days'::interval) downloads_180_days, + count(dl.id) filter (where dl.created_at > now() - '90 days'::interval) downloads_90_days, + count(dl.id) filter (where dl.created_at > now() - '30 days'::interval) downloads_30_day + from + app.downloads dl + group by + dl.package_id; + + +-- High frequency refresh for debugging +select cron.schedule( + 'refresh download metrics', + '*/30 * * * *', + 'refresh materialized view public.download_metrics;' +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230411104448_download_metrics_computed_relation.sql b/crates/pgls_pretty_print/tests/data/multi/20230411104448_download_metrics_computed_relation.sql new file mode 100644 index 000000000..6e285bf4e --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230411104448_download_metrics_computed_relation.sql @@ -0,0 +1,8 @@ +create or replace function public.download_metrics(public.packages) +returns setof public.download_metrics rows 1 +language sql stable +as $$ + select * + from public.download_metrics dm + where dm.package_id = $1.id; +$$; diff --git a/crates/pgls_pretty_print/tests/data/multi/20230411175952_langchain-embedding_search.sql b/crates/pgls_pretty_print/tests/data/multi/20230411175952_langchain-embedding_search.sql new file mode 100644 index 000000000..3bc16b5fa --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230411175952_langchain-embedding_search.sql @@ -0,0 +1,139 @@ +insert into app.packages( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values ( + 'langchain', + 'embedding_search', + 'Search document embeddings', + true, + '{pg_vector}' +); + + + + +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_name = 'langchain-embedding_search'), +(1,0,0), +$pkg$ +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = 'vector' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception '"langchain-embedding_search" requires "vector"' + using hint = 'Run "create extension vector" and try again'; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +$pkg$, + +$description_md$ +# embedding_search + +[LangChain](https://js.langchain.com/docs/) is a framework for developing applications powered by language models with a plugable architecture. + +`langchain-embedding_search` uses a Supabase Postgres database as its vector store. + +## Installation + +```sql +select dbdev.install('langchain-embedding_search'); +create extension if not exists vector; +create extension "langchain-embedding_search" + schema public + version '1.0.0'; +``` +Note: + +`vector` is a dependency of `langchain-embedding_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromTexts( + ["Hello world", "Bye bye", "What's this?"], + [{ id: 2 }, { id: 1 }, { id: 3 }], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + const resultOne = await vectorStore.similaritySearch("Hello world", 1); + + console.log(resultOne); +}; +``` + +For more details, checkout the LangChain Supabase integration docs: https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase +$description_md$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230411175953_langchain-hybrid_search.sql b/crates/pgls_pretty_print/tests/data/multi/20230411175953_langchain-hybrid_search.sql new file mode 100644 index 000000000..7012607f7 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230411175953_langchain-hybrid_search.sql @@ -0,0 +1,160 @@ +insert into app.packages( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values ( + 'langchain', + 'hybrid_search', + 'Search documents by embedding and full text', + true, + '{pg_vector}' +); + + + + +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_name = 'langchain-hybrid_search'), +(1,0,0), +$pkg$ +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = 'vector' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception '"langchain-hybrid_search" requires "vector"' + using hint = 'Run "create extension vector" and try again'; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to similarity search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +-- Create a function to keyword search for documents +create function kw_match_documents(query_text text, match_count int) +returns table (id bigint, content text, metadata jsonb, similarity real) +as $$ + +begin +return query execute +format(' + select + id, content, metadata, ts_rank(to_tsvector(content), plainto_tsquery($1)) as similarity + from + documents + where + to_tsvector(content) @@ plainto_tsquery($1) + order by + similarity desc + limit $2 +') +using query_text, match_count; +end; +$$ language plpgsql; + +$pkg$, + +$description_md$ +# hybrid_search + +Langchain supports hybrid search with a Supabase Postgres database. The hybrid search combines the postgres pgvector extension (similarity search) and Full-Text Search (keyword search) to retrieve documents. You can add documents via SupabaseVectorStore addDocuments function. SupabaseHybridKeyWordSearch accepts embedding, supabase client, number of results for similarity search, and number of results for keyword search as parameters. The getRelevantDocuments function produces a list of documents that has duplicates removed and is sorted by relevance score. + +## Installation + +```sql +select dbdev.install('langchain-hybrid_search'); +create extension if not exists vector; +create extension "langchain-hybrid_search" + schema public + version '1.0.0'; +``` +Note: + +`vector` is a dependency of `langchain-hybrid_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; +import { SupabaseHybridSearch } from "langchain/retrievers/supabase"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const embeddings = new OpenAIEmbeddings(); + + const retriever = new SupabaseHybridSearch(embeddings, { + client, + // Below are the defaults, expecting that you set up your supabase table and functions according to the guide above. Please change if necessary. + similarityK: 2, + keywordK: 2, + tableName: "documents", + similarityQueryName: "match_documents", + keywordQueryName: "kw_match_documents", + }); + + const results = await retriever.getRelevantDocuments("hello bye"); + + console.log(results); +}; +``` + +For more details, checkout the LangChain Supabase Hybrid Search docs: https://js.langchain.com/docs/modules/indexes/retrievers/supabase-hybrid +$description_md$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230413130634_popular_packages_function.sql b/crates/pgls_pretty_print/tests/data/multi/20230413130634_popular_packages_function.sql new file mode 100644 index 000000000..bb08c80e7 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230413130634_popular_packages_function.sql @@ -0,0 +1,11 @@ +create or replace function public.popular_packages() +returns setof public.packages +language sql stable +as $$ + select * from public.packages p + order by ( + select (dm.downloads_30_day * 5) + (dm.downloads_90_days * 2) + dm.downloads_180_days + from public.download_metrics dm + where dm.package_id = p.id + ) desc nulls last, p.created_at desc; +$$; diff --git a/crates/pgls_pretty_print/tests/data/multi/20230413140356_update_profile_function.sql b/crates/pgls_pretty_print/tests/data/multi/20230413140356_update_profile_function.sql new file mode 100644 index 000000000..76472471b --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230413140356_update_profile_function.sql @@ -0,0 +1,22 @@ +create or replace function public.update_profile( + handle app.valid_name, + display_name text default null, + bio text default null +) +returns void +language plpgsql +as $$ + declare + v_is_org boolean; + begin + update app.accounts a + set display_name = coalesce($2, a.display_name), + bio = coalesce($3, a.bio) + where a.handle = $1; + + update app.organizations o + set display_name = coalesce($2, o.display_name), + bio = coalesce($3, o.bio) + where o.handle = $1; + end; +$$; diff --git a/crates/pgls_pretty_print/tests/data/multi/20230417141004_dbdev_short_desc_typo.sql b/crates/pgls_pretty_print/tests/data/multi/20230417141004_dbdev_short_desc_typo.sql new file mode 100644 index 000000000..d9b2a060e --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230417141004_dbdev_short_desc_typo.sql @@ -0,0 +1,4 @@ +update app.packages +-- Fix typo in spelling of "packages" +set control_description = 'Install packages from the database.dev registry' +where handle = 'supabase' and partial_name = 'dbdev'; diff --git a/crates/pgls_pretty_print/tests/data/multi/20230508165641_packages_order_version.sql b/crates/pgls_pretty_print/tests/data/multi/20230508165641_packages_order_version.sql new file mode 100644 index 000000000..59ddb2cba --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230508165641_packages_order_version.sql @@ -0,0 +1,20 @@ +create or replace view public.packages as + select + pa.id, + pa.package_name, + pa.handle, + pa.partial_name, + newest_ver.version as latest_version, + newest_ver.description_md, + pa.control_description, + pa.control_requires, + pa.created_at + from + app.packages pa, + lateral ( + select * + from app.package_versions pv + where pv.package_id = pa.id + order by pv.version_struct desc + limit 1 + ) newest_ver; diff --git a/crates/pgls_pretty_print/tests/data/multi/20230508175952_langchain-embedding_search-1_1_0.sql b/crates/pgls_pretty_print/tests/data/multi/20230508175952_langchain-embedding_search-1_1_0.sql new file mode 100644 index 000000000..d3147f643 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230508175952_langchain-embedding_search-1_1_0.sql @@ -0,0 +1,167 @@ +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_name = 'langchain-embedding_search'), +(1,1,0), +$pkg$ +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = 'vector' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception '"langchain-embedding_search" requires "vector"' + using hint = 'Run "create extension vector" and try again'; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int, + filter jsonb DEFAULT '{}' +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + where metadata @> filter + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +$pkg$, + +$description_md$ +# embedding_search + +[LangChain](https://js.langchain.com/docs/) is a framework for developing applications powered by language models with a plugable architecture. + +`langchain-embedding_search` uses a Supabase Postgres database as its vector store. + +## Installation + +```sql +select dbdev.install('langchain-embedding_search'); +create extension if not exists vector; +create extension "langchain-embedding_search" + schema public + version '1.1.0'; +``` +Note: + +`vector` is a dependency of `langchain-embedding_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +### Standard Usage + +The below example shows how to perform a basic similarity search with Supabase: + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromTexts( + ["Hello world", "Bye bye", "What's this?"], + [{ id: 2 }, { id: 1 }, { id: 3 }], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + const resultOne = await vectorStore.similaritySearch("Hello world", 1); + + console.log(resultOne); +}; +``` + +### Metadata Filtering + +Given the above `match_documents` Postgres function, you can also pass a filter parameter to only documents with a specific metadata field value. + +```js +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; + +// First, follow set-up instructions at +// https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromTexts( + ["Hello world", "Hello world", "Hello world"], + [{ user_id: 2 }, { user_id: 1 }, { user_id: 3 }], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + const result = await vectorStore.similaritySearch("Hello world", 1, { + user_id: 3, + }); + + console.log(result); +}; +``` + +For more details, checkout the LangChain Supabase integration docs: https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase +$description_md$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230508175953_langchain-hybrid_search-1_1_0.sql b/crates/pgls_pretty_print/tests/data/multi/20230508175953_langchain-hybrid_search-1_1_0.sql new file mode 100644 index 000000000..3ac370c04 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230508175953_langchain-hybrid_search-1_1_0.sql @@ -0,0 +1,144 @@ +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_name = 'langchain-hybrid_search'), +(1,1,0), +$pkg$ +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = 'vector' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception '"langchain-hybrid_search" requires "vector"' + using hint = 'Run "create extension vector" and try again'; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to similarity search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int, + filter jsonb DEFAULT '{}' +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + where metadata @> filter + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +-- Create a function to keyword search for documents +create function kw_match_documents(query_text text, match_count int) +returns table (id bigint, content text, metadata jsonb, similarity real) +as $$ + +begin +return query execute +format(' + select + id, content, metadata, ts_rank(to_tsvector(content), plainto_tsquery($1)) as similarity + from + documents + where + to_tsvector(content) @@ plainto_tsquery($1) + order by + similarity desc + limit $2 +') +using query_text, match_count; +end; +$$ language plpgsql; + +$pkg$, + +$description_md$ +# hybrid_search + +Langchain supports hybrid search with a Supabase Postgres database. The hybrid search combines the postgres pgvector extension (similarity search) and Full-Text Search (keyword search) to retrieve documents. You can add documents via SupabaseVectorStore addDocuments function. SupabaseHybridKeyWordSearch accepts embedding, supabase client, number of results for similarity search, and number of results for keyword search as parameters. The getRelevantDocuments function produces a list of documents that has duplicates removed and is sorted by relevance score. + +## Installation + +```sql +select dbdev.install('langchain-hybrid_search'); +create extension if not exists vector; +create extension "langchain-hybrid_search" + schema public + version '1.1.0'; +``` +Note: + +`vector` is a dependency of `langchain-hybrid_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; +import { SupabaseHybridSearch } from "langchain/retrievers/supabase"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const embeddings = new OpenAIEmbeddings(); + + const retriever = new SupabaseHybridSearch(embeddings, { + client, + // Below are the defaults, expecting that you set up your supabase table and functions according to the guide above. Please change if necessary. + similarityK: 2, + keywordK: 2, + tableName: "documents", + similarityQueryName: "match_documents", + keywordQueryName: "kw_match_documents", + }); + + const results = await retriever.getRelevantDocuments("hello bye"); + + console.log(results); +}; +``` + +For more details, checkout the LangChain Supabase Hybrid Search docs: https://js.langchain.com/docs/modules/indexes/retrievers/supabase-hybrid +$description_md$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230622212339_langchain_headerkit_config_dump.sql b/crates/pgls_pretty_print/tests/data/multi/20230622212339_langchain_headerkit_config_dump.sql new file mode 100644 index 000000000..859eac621 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230622212339_langchain_headerkit_config_dump.sql @@ -0,0 +1,37 @@ + +-- update dependencies for langchain + +UPDATE app.packages SET control_requires = '{vector}' WHERE handle = 'langchain'; + +INSERT INTO app.package_upgrades(package_id, from_version_struct, to_version_struct, sql) +VALUES ( +(SELECT id FROM app.packages WHERE handle = 'langchain' AND partial_name = 'embedding_search'), +(1,1,0), +(1,1,1), +$langchain$ +SELECT pg_extension_config_dump('documents', ''); +SELECT pg_extension_config_dump('documents_id_seq', ''); +$langchain$ +); + +INSERT INTO app.package_upgrades(package_id, from_version_struct, to_version_struct, sql) +VALUES ( +(SELECT id FROM app.packages WHERE handle = 'langchain' AND partial_name = 'hybrid_search'), +(1,1,0), +(1,1,1), +$langchain$ +SELECT pg_extension_config_dump('documents', ''); +SELECT pg_extension_config_dump('documents_id_seq', ''); +$langchain$ +); + +INSERT INTO app.package_upgrades(package_id, from_version_struct, to_version_struct, sql) +VALUES ( +(SELECT id FROM app.packages WHERE partial_name = 'pg_headerkit'), +(1,0,0), +(1,0,1), +$hdr$ +SELECT pg_extension_config_dump('hdr.allow_list', ''); +SELECT pg_extension_config_dump('hdr.deny_list', ''); +$hdr$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230623181432_dbdev_supports_multiple_versions.sql b/crates/pgls_pretty_print/tests/data/multi/20230623181432_dbdev_supports_multiple_versions.sql new file mode 100644 index 000000000..78542a38d --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230623181432_dbdev_supports_multiple_versions.sql @@ -0,0 +1,284 @@ +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_name = 'supabase-dbdev'), +(0,0,3), +$pkg$ + +create schema dbdev; + +create or replace function dbdev.install(package_name text) + returns bool + language plpgsql +as $$ +declare + -- Endpoint + base_url text = 'https://api.database.dev/rest/v1/'; + apikey text = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyYndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzIsImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJrzM0AQKsu_5k134s'; + + http_ext_schema regnamespace = extnamespace::regnamespace from pg_catalog.pg_extension where extname = 'http' limit 1; + pgtle_is_available bool = true from pg_catalog.pg_extension where extname = 'pg_tle' limit 1; + -- HTTP respones + rec jsonb; + status int; + contents json; + + -- Install Record + rec_sql text; + rec_ver text; + rec_from_ver text; + rec_to_ver text; + rec_package_name text; + rec_description text; + rec_requires text[]; +begin + + if http_ext_schema is null then + raise exception using errcode='22000', message=format('dbdev requires the http extension and it is not available'); + end if; + + if pgtle_is_available is null then + raise exception using errcode='22000', message=format('dbdev requires the pgtle extension and it is not available'); + end if; + + ------------------- + -- Base Versions -- + ------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + 'GET', + format( + '%spackage_versions?select=package_name,version,sql,control_description,control_requires&limit=50&package_name=eq.%s', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + ('apiKey', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> 'status')::int; + contents = to_json(rec ->> 'content') #>> '{}'; + + if status <> 200 then + raise notice using errcode='22000', message=format('DBDEV INFO: %s', contents); + raise exception using errcode='22000', message=format('Non-200 response code while loading versions from dbdev'); + end if; + + if contents is null or json_typeof(contents) <> 'array' or json_array_length(contents) = 0 then + raise exception using errcode='22000', message=format('No versions for package named named %s', package_name); + end if; + + for rec_package_name, rec_ver, rec_sql, rec_description, rec_requires in select + (r ->> 'package_name'), + (r ->> 'version'), + (r ->> 'sql'), + (r ->> 'control_description'), + array(select json_array_elements_text((r -> 'control_requires'))) + from + json_array_elements(contents) as r + loop + + -- Install the primary version + if not exists ( + select true + from pgtle.available_extensions() + where + name = rec_package_name + ) then + perform pgtle.install_extension(rec_package_name, rec_ver, rec_package_name, rec_sql, rec_requires); + end if; + + -- Install other available versions + if not exists ( + select true + from pgtle.available_extension_versions() + where + name = rec_package_name + and version = rec_ver + ) then + perform pgtle.install_extension_version_sql(rec_package_name, rec_ver, rec_sql); + end if; + + end loop; + + ---------------------- + -- Upgrade Versions -- + ---------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + 'GET', + format( + '%spackage_upgrades?select=package_name,from_version,to_version,sql&limit=50&package_name=eq.%s', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + ('apiKey', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> 'status')::int; + contents = to_json(rec ->> 'content') #>> '{}'; + + if status <> 200 then + raise notice using errcode='22000', message=format('DBDEV INFO: %s', contents); + raise exception using errcode='22000', message=format('Non-200 response code while loading upgrade pathes from dbdev'); + end if; + + if json_typeof(contents) <> 'array' then + raise exception using errcode='22000', message=format('Invalid response from dbdev upgrade pathes'); + end if; + + for rec_package_name, rec_from_ver, rec_to_ver, rec_sql in select + (r ->> 'package_name'), + (r ->> 'from_version'), + (r ->> 'to_version'), + (r ->> 'sql') + from + json_array_elements(contents) as r + loop + + if not exists ( + select true + from pgtle.extension_update_paths(rec_package_name) + where + source = rec_from_ver + and target = rec_to_ver + and path is not null + ) then + perform pgtle.install_update_path(rec_package_name, rec_from_ver, rec_to_ver, rec_sql); + end if; + end loop; + + -------------------------- + -- Send Download Notice -- + -------------------------- + -- Notifies dbdev that a package has been downloaded and records IP + user agent so we can compute unique download counts + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + 'POST', + format( + '%srpc/register_download', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$ + ), + array[ + ('apiKey', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header, + ('x-client-info', 'dbdev/0.0.2')::http_header + ], + 'application/json', + json_build_object('package_name', $stmt$ || pg_catalog.quote_literal($1) || $stmt$)::text + ) + ) x + limit 1; $stmt$ + into rec; + + return true; +end; +$$; + +$pkg$, +$description$ +# dbdev + +dbdev is the SQL client for database.new and is the primary way end users interact with the package (pglet) registry. + +dbdev can be used to load packages from the registry. For example: + +```sql +-- Load the package from the package index +select dbdev.install('olirice-index_advisor'); +``` +Where `olirice` is the handle of the author and `index_advisor` is the name of the pglet. + +Once installed, pglets are visible in PostgreSQL as extensions. At that point they can be enabled with standard Postgres commands i.e. the `create extension` + +To improve reproducibility, we recommend __always__ specifying the package version in your `create extension` statements. + +For example: +```sql +-- Enable the extension +create extension "olirice-index_advisor" + schema 'public' + version '0.1.0'; +``` + +Which creates all tables/indexes/functions/etc specified by the extension. + +## How to Install + +The in-database SQL client for the package registry is named `dbdev`. You can bootstrap the client with: + +```sql +/*--------------------- +---- install dbdev ---- +---------------------- +Requires: + - pg_tle: https://github.com/aws/pg_tle + - pgsql-http: https://github.com/pramsey/pgsql-http +*/ +create extension if not exists http with schema extensions; +create extension if not exists pg_tle; +select pgtle.uninstall_extension_if_exists('supabase-dbdev'); +drop extension if exists "supabase-dbdev"; +select + pgtle.install_extension( + 'supabase-dbdev', + resp.contents ->> 'version', + 'PostgreSQL package manager', + resp.contents ->> 'sql' + ) +from http( + ( + 'GET', + 'https://api.database.dev/rest/v1/' + || 'package_versions?select=sql,version' + || '&package_name=eq.supabase-dbdev' + || '&order=version.desc' + || '&limit=1', + array[ + ( + 'apiKey', + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJp' + || 'c3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyY' + || 'ndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzI' + || 'sImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJ' + || 'rzM0AQKsu_5k134s' + )::http_header + ], + null, + null + ) +) x, +lateral ( + select + ((row_to_json(x) -> 'content') #>> '{}')::json -> 0 +) resp(contents); +create extension "supabase-dbdev"; +select dbdev.install('supabase-dbdev'); +drop extension if exists "supabase-dbdev"; +create extension "supabase-dbdev"; +``` + +With the client ready, search for packages on [database.dev](database.dev) and install them with + +```sql +select dbdev.install('handle-package_name'); +create extension "handle-package_name" + schema 'public' + version '1.2.3'; +``` +$description$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230829125510_fix_view_permissions.sql b/crates/pgls_pretty_print/tests/data/multi/20230829125510_fix_view_permissions.sql new file mode 100644 index 000000000..6eac41113 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230829125510_fix_view_permissions.sql @@ -0,0 +1,6 @@ +alter view public.accounts set (security_invoker=true); +alter view public.organizations set (security_invoker=true); +alter view public.members set (security_invoker=true); +alter view public.packages set (security_invoker=true); +alter view public.package_versions set (security_invoker=true); +alter view public.package_upgrades set (security_invoker=true); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230830083255_olirice-index_advisor-0_2_0.sql b/crates/pgls_pretty_print/tests/data/multi/20230830083255_olirice-index_advisor-0_2_0.sql new file mode 100644 index 000000000..058419ba9 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230830083255_olirice-index_advisor-0_2_0.sql @@ -0,0 +1,343 @@ +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_name = 'olirice-index_advisor'), +(0,2,0), +$pkg$ + +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + hypopg_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = 'hypopg' + and installed_version is not null + ); + begin + + if not hypopg_exists then + raise + exception '"olirice-index_advisor" requires "hypopg"' + using hint = 'Run "create extension hypopg" and try again'; + end if; + end +$$; + +create or replace function index_advisor( + query text +) + returns table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[], + errors text[] + ) + volatile + language plpgsql + as $$ +declare + n_args int; + prepared_statement_name text = 'index_advisor_working_statement'; + hypopg_schema_name text = (select extnamespace::regnamespace::text from pg_extension where extname = 'hypopg'); + explain_plan_statement text; + error_message text; + rec record; + plan_initial jsonb; + plan_final jsonb; + statements text[] = '{}'; +begin + + -- Remove comment lines (its common that they contain semicolons) + query := trim( + regexp_replace( + regexp_replace( + regexp_replace(query,'\/\*.+\*\/', '', 'g'), + '--[^\r\n]*', ' ', 'g'), + '\s+', ' ', 'g') + ); + + -- Remove trailing semicolon + query := regexp_replace(query, ';\s*$', ''); + + begin + -- Disallow multiple statements + if query ilike '%;%' then + raise exception 'Query must not contain a semicolon'; + end if; + + -- Hack to support PostgREST because the prepared statement for args incorrectly defaults to text + query := replace(query, 'WITH pgrst_payload AS (SELECT $1 AS json_data)', 'WITH pgrst_payload AS (SELECT $1::json AS json_data)'); + + -- Create a prepared statement for the given query + deallocate all; + execute format('prepare %I as %s', prepared_statement_name, query); + + -- Detect how many arguments are present in the prepared statement + n_args = ( + select + coalesce(array_length(parameter_types, 1), 0) + from + pg_prepared_statements + where + name = prepared_statement_name + limit + 1 + ); + + -- Create a SQL statement that can be executed to collect the explain plan + explain_plan_statement = format( + 'set local plan_cache_mode = force_generic_plan; explain (format json) execute %I%s', + --'explain (format json) execute %I%s', + prepared_statement_name, + case + when n_args = 0 then '' + else format( + '(%s)', array_to_string(array_fill('null'::text, array[n_args]), ',') + ) + end + ); + + -- Store the query plan before any new indexes + execute explain_plan_statement into plan_initial; + + -- Create possible indexes + for rec in ( + with extension_regclass as ( + select + distinct objid as oid + from + pg_catalog.pg_depend + where + deptype = 'e' + ) + select + pc.relnamespace::regnamespace::text as schema_name, + pc.relname as table_name, + pa.attname as column_name, + format( + 'select %I.hypopg_create_index($i$create index on %I.%I(%I)$i$)', + hypopg_schema_name, + pc.relnamespace::regnamespace::text, + pc.relname, + pa.attname + ) hypopg_statement + from + pg_catalog.pg_class pc + join pg_catalog.pg_attribute pa + on pc.oid = pa.attrelid + left join extension_regclass er + on pc.oid = er.oid + left join pg_catalog.pg_index pi + on pc.oid = pi.indrelid + and (select array_agg(x) from unnest(pi.indkey) v(x)) = array[pa.attnum] + and pi.indexprs is null -- ignore expression indexes + and pi.indpred is null -- ignore partial indexes + where + pc.relnamespace::regnamespace::text not in ( -- ignore schema list + 'pg_catalog', 'pg_toast', 'information_schema' + ) + and er.oid is null -- ignore entities owned by extensions + and pc.relkind in ('r', 'm') -- regular tables, and materialized views + and pc.relpersistence = 'p' -- permanent tables (not unlogged or temporary) + and pa.attnum > 0 + and not pa.attisdropped + and pi.indrelid is null + and pa.atttypid in (20,16,1082,1184,1114,701,23,21,700,1083,2950,1700,25,18,1042,1043) + ) + loop + -- Create the hypothetical index + execute rec.hypopg_statement; + end loop; + + -- Create a prepared statement for the given query + -- The original prepared statement MUST be dropped because its plan is cached + execute format('deallocate %I', prepared_statement_name); + execute format('prepare %I as %s', prepared_statement_name, query); + + -- Store the query plan after new indexes + execute explain_plan_statement into plan_final; + + --raise notice '%', plan_final; + + -- Idenfity referenced indexes in new plan + execute format( + 'select + coalesce(array_agg(hypopg_get_indexdef(indexrelid) order by indrelid, indkey::text), $i${}$i$::text[]) + from + %I.hypopg() + where + %s ilike ($i$%%$i$ || indexname || $i$%%$i$) + ', + hypopg_schema_name, + quote_literal(plan_final)::text + ) into statements; + + -- Reset all hypothetical indexes + perform hypopg_reset(); + + -- Reset prepared statements + deallocate all; + + return query values ( + (plan_initial -> 0 -> 'Plan' -> 'Startup Cost'), + (plan_final -> 0 -> 'Plan' -> 'Startup Cost'), + (plan_initial -> 0 -> 'Plan' -> 'Total Cost'), + (plan_final -> 0 -> 'Plan' -> 'Total Cost'), + statements::text[], + array[]::text[] + ); + return; + + exception when others then + get stacked diagnostics error_message = MESSAGE_TEXT; + + return query values ( + null::jsonb, + null::jsonb, + null::jsonb, + null::jsonb, + array[]::text[], + array[error_message]::text[] + ); + return; + end; + +end; +$$; + +$pkg$, + +$description_md$ + +# index_advisor + +`index_advisor` is an extension for recommending indexes to improve query performance. + +## Installation + +Note: + +`hypopg` is a dependency of index_advisor. +Dependency resolution is currently under development. +In the future it will not be necessary to manually create dependencies. + + +```sql +select dbdev.install('olirice-index_advisor'); +create extension if not exists hypopg; +create extension "olirice-index_advisor" version '0.2.0'; +``` + +Features: +- Supports generic parameters e.g. `$1`, `$2` +- Supports materialized views +- Identifies tables/columns obfuscaed by views + + +## API + +#### Description +For a given *query*, searches for a set of SQL DDL `create index` statements that improve the query's execution time; + +#### Signature +```sql +index_advisor(query text) +returns + table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[], + errors text[] + ) +``` + +## Usage + +For a minimal example, the `index_advisor` function can be given a single table query with a filter on an unindexed column. + +```sql +create extension if not exists index_advisor cascade; + +create table book( + id int primary key, + title text not null +); + +select + * +from + index_advisor('select book.id from book where title = $1'); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements | errors +---------------------+--------------------+-------------------+------------------+-----------------------------------------------------+-------- + 0.00 | 1.17 | 25.88 | 6.40 | {"CREATE INDEX ON public.book USING btree (title)"},| {} +(1 row) +``` + +More complex queries may generate additional suggested indexes + +```sql +create extension if not exists index_advisor cascade; + +create table author( + id serial primary key, + name text not null +); + +create table publisher( + id serial primary key, + name text not null, + corporate_address text +); + +create table book( + id serial primary key, + author_id int not null references author(id), + publisher_id int not null references publisher(id), + title text +); + +create table review( + id serial primary key, + book_id int references book(id), + body text not null +); + +select + * +from + index_advisor(' + select + book.id, + book.title, + publisher.name as publisher_name, + author.name as author_name, + review.body review_body + from + book + join publisher + on book.publisher_id = publisher.id + join author + on book.author_id = author.id + join review + on book.id = review.book_id + where + author.id = $1 + and publisher.id = $2 + '); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements | errors +---------------------+--------------------+-------------------+------------------+-----------------------------------------------------------+-------- + 27.26 | 12.77 | 68.48 | 42.37 | {"CREATE INDEX ON public.book USING btree (author_id)", | {} + "CREATE INDEX ON public.book USING btree (publisher_id)", + "CREATE INDEX ON public.review USING btree (book_id)"} +(3 rows) +``` +$description_md$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230831172915_allow_anon_access_to_package_views.sql b/crates/pgls_pretty_print/tests/data/multi/20230831172915_allow_anon_access_to_package_views.sql new file mode 100644 index 000000000..a2b9ee8ff --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230831172915_allow_anon_access_to_package_views.sql @@ -0,0 +1,3 @@ +alter view public.packages set (security_invoker=false); +alter view public.package_versions set (security_invoker=false); +alter view public.package_upgrades set (security_invoker=false); diff --git a/crates/pgls_pretty_print/tests/data/multi/20230906110845_access_token.sql b/crates/pgls_pretty_print/tests/data/multi/20230906110845_access_token.sql new file mode 100644 index 000000000..98daa00cb --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230906110845_access_token.sql @@ -0,0 +1,196 @@ +create table app.access_tokens( + id uuid primary key default gen_random_uuid(), + user_id uuid not null references auth.users(id) on delete cascade, + token_hash bytea not null, + token_name text not null check (length(token_name) <= 64), + plaintext_suffix text not null check (length(plaintext_suffix) = 4), + created_at timestamptz not null default now() +); + +grant insert + (id, user_id, token_hash, token_name, plaintext_suffix) + on app.access_tokens + to authenticated; + +grant delete + on app.access_tokens + to authenticated; + +alter table app.access_tokens enable row level security; + +create policy access_tokens_select_policy + on app.access_tokens + as permissive + for select + to public + using ( auth.uid() = user_id ); + +create policy access_tokens_insert_policy + on app.access_tokens + as permissive + for insert + to authenticated + with check ( auth.uid() = user_id ); + +create policy access_tokens_delete_policy + on app.access_tokens + as permissive + for delete + to authenticated + using ( auth.uid() = user_id ); + +create or replace function app.base64url_encode(input bytea) + returns text + language plpgsql + strict +as $$ +begin + return replace(replace(encode(input, 'base64'), '/', '_'), '+', '-'); +end; +$$; + +create or replace function app.base64url_decode(input text) + returns text + language plpgsql + strict +as $$ +begin + return decode(replace(replace(input, '-', '+'), '_', '/'), 'base64'); +end; +$$; + +create or replace function public.new_access_token( + token_name text +) + returns text + language plpgsql + strict +as $$ +<> +declare + account app.accounts = account from app.accounts account where id = auth.uid(); + -- Why 21 random bytes? We are shooting for 128 bit (16 bytes) entropy. But we + -- also show three bytes as plaintext. That takes us to a total 19 bytes. + -- We add two bytes to make sure that the base64 encoded bytes don't have any + -- padding, which makes it a little bit nicer to look at. That makes 21. + token bytea = gen_random_bytes(21); + token_hash bytea = sha256(token); + -- Total length of the base64 encoded token is 21 * 8 / 6 = 28 + token_text text = app.base64url_encode(token); + token_id uuid; + -- Last 4 base64 encoded character are shown in the suffix + plaintext_suffix text = substring(token_text from 25); +begin + insert into app.access_tokens(user_id, token_hash, token_name, plaintext_suffix) + values (account.id, token_hash, token_name, fn.plaintext_suffix) returning id into token_id; + + -- String returned has a length 64 + return 'dbd_' || replace(token_id::text, '-', '') || token_text; +end; +$$; + +create type app.access_token_struct as ( + id uuid, + token_name text, + masked_token text, + created_at timestamptz +); + +create or replace function public.get_access_tokens() + returns setof app.access_token_struct + language plpgsql + strict +as $$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); +begin + return query + select id, token_name, + 'dbd_' || + substring(at.id::text from 1 for 4) || + repeat('•', 52) || + at.plaintext_suffix as masked_token, + created_at + from app.access_tokens at + where at.user_id = account.id; +end; +$$; + +create or replace function public.delete_access_token( + token_id uuid +) + returns void + language plpgsql + strict +as $$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); +begin + delete from app.access_tokens at + where at.user_id = account.id and at.id = token_id; +end; +$$; + +create type app.user_id_and_token_hash as ( + user_id uuid, + token_hash bytea +); + +create or replace function public.redeem_access_token( + access_token text +) + returns text + language plpgsql + security definer + strict +as $$ +declare + token_id uuid; + token bytea; + tokens_row app.user_id_and_token_hash; + token_valid boolean; + now timestamp; + one_hour_from_now timestamp; + issued_at int; + expiry_at int; + jwt_secret text; +begin + -- validate access token + if length(access_token) != 64 then + raise exception 'Invalid token'; + end if; + + if substring(access_token from 1 for 4) != 'dbd_' then + raise exception 'Invalid token'; + end if; + + token_id := substring(access_token from 5 for 32)::uuid; + token := app.base64url_decode(substring(access_token from 37)); + + select t.user_id, t.token_hash + into tokens_row + from app.access_tokens t + where t.id = token_id; + + -- TODO: do a constant time comparison + if tokens_row.token_hash != sha256(token) then + raise exception 'Invalid token'; + end if; + + -- Generate JWT token + now := current_timestamp; + one_hour_from_now := now + interval '1 hour'; + issued_at := date_part('epoch', now); + expiry_at := date_part('epoch', one_hour_from_now); + jwt_secret := current_setting('app.settings.jwt_secret', true); + + return sign(json_build_object( + 'aud', 'authenticated', + 'role', 'authenticated', + 'iss', 'database.dev', + 'sub', tokens_row.user_id, + 'iat', issued_at, + 'exp', expiry_at + ), jwt_secret); +end; +$$; diff --git a/crates/pgls_pretty_print/tests/data/multi/20230906111353_publish_package.sql b/crates/pgls_pretty_print/tests/data/multi/20230906111353_publish_package.sql new file mode 100644 index 000000000..ccd14a2ca --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20230906111353_publish_package.sql @@ -0,0 +1,106 @@ +grant insert (partial_name, handle, control_description) + on app.packages + to authenticated; + +grant update (control_description) + on app.packages + to authenticated; + +create policy packages_update_policy + on app.packages + as permissive + for update + to authenticated + using ( app.is_package_maintainer(auth.uid(), id) ); + +create or replace function public.publish_package( + package_name app.valid_name, + package_description varchar(1000) +) + returns void + language plpgsql +as $$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); +begin + if account.handle is null then + raise exception 'user not logged in'; + end if; + + insert into app.packages(handle, partial_name, control_description) + values (account.handle, package_name, package_description) + on conflict on constraint packages_handle_partial_name_key + do update + set control_description = excluded.control_description; +end; +$$; + +create or replace function public.publish_package_version( + package_name app.valid_name, + version_source text, + version_description text, + version text +) + returns uuid + language plpgsql +as $$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); + package_id uuid; + version_id uuid; +begin + if account.handle is null then + raise exception 'user not logged in'; + end if; + + select ap.id + from app.packages ap + where ap.handle = account.handle and ap.partial_name = publish_package_version.package_name + into package_id; + + begin + insert into app.package_versions(package_id, version_struct, sql, description_md) + values (package_id, app.text_to_semver(version), version_source, version_description) + returning id into version_id; + + return version_id; + exception when unique_violation then + return null; + end; +end; +$$; + +create or replace function public.publish_package_upgrade( + package_name app.valid_name, + upgrade_source text, + from_version text, + to_version text +) + returns uuid + language plpgsql +as $$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); + package_id uuid; + upgrade_id uuid; +begin + if account.handle is null then + raise exception 'user not logged in'; + end if; + + select ap.id + from app.packages ap + where ap.handle = account.handle and ap.partial_name = publish_package_upgrade.package_name + into package_id; + + begin + insert into app.package_upgrades(package_id, from_version_struct, to_version_struct, sql) + values (package_id, app.text_to_semver(from_version), app.text_to_semver(to_version), upgrade_source) + returning id into upgrade_id; + + return upgrade_id; + exception when unique_violation then + return null; + end; +end; +$$; diff --git a/crates/pgls_pretty_print/tests/data/multi/20231110061036_allow_publishing_relocatable_and_requires.sql b/crates/pgls_pretty_print/tests/data/multi/20231110061036_allow_publishing_relocatable_and_requires.sql new file mode 100644 index 000000000..8cea28af5 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20231110061036_allow_publishing_relocatable_and_requires.sql @@ -0,0 +1,167 @@ +-- A list of extensions which are allowed in the requires key of the control file +create table app.allowed_extensions ( + name text primary key +); + +insert into app.allowed_extensions (name) +values +-- extensions available on Supabase + ('citext'), + ('pg_cron'), + ('pg_graphql'), + ('pg_stat_statements'), + ('pg_trgm'), + ('pg_crypto'), + ('pg_jwt'), + ('pg_sodium'), + ('plpgsql'), + ('uuid-ossp'), + ('address_standardizer'), + ('address_standardizer_data_us'), + ('autoinc'), + ('bloom'), + ('btree_gin'), + ('btree_gist'), + ('cube'), + ('dblink'), + ('dict_int'), + ('dict_xsyn'), + ('earthdistance'), + ('fuzzystrmatch'), + ('hstore'), + ('http'), + ('hypopg'), + ('insert_username'), + ('intarray'), + ('isn'), + ('ltree'), + ('moddatetime'), + ('pg_hashids'), + ('pg_jsonschema'), + ('pg_net'), + ('pg_repack'), + ('pg_stat_monitor'), + ('pg_walinspect'), + ('pgaudit'), + ('pgroonga'), + ('pgroonga_database'), + ('pgrouting'), + ('pgrowlocks'), + ('pgtap'), + ('plcoffee'), + ('pljava'), + ('plls'), + ('plpgsql_check'), + ('plv8'), + ('postgis'), + ('postgis_raster'), + ('postgis_sfcgal'), + ('postgis_tiger_geocoder'), + ('postgis_topology'), + ('postgres_fdw'), + ('refint'), + ('rum'), + ('seg'), + ('sslinfo'), + ('supautils'), + ('tablefunc'), + ('tcn'), + ('timescaledb'), + ('tsm_system_rows'), + ('tsm_system_time'), + ('unaccent'), + ('vector'), + ('wrappers'), + +-- extensions available on AWS (except those already in Supabase) +-- full list here: https://docs.aws.amazon.com/AmazonRDS/latest/PostgreSQLReleaseNotes/postgresql-extensions.html + ('amcheck'), + ('aws_commons'), + ('aws_lambda'), + ('aws_s3'), + ('bool_plperl'), + ('decoder_raw'), + ('h3-pg'), + ('hll'), + ('hstore_plperl'), + ('intagg'), + ('ip4r'), + ('jsonb_plperl'), + ('lo'), + ('log_fdw'), + ('mysql_fdw'), + ('old_snapshot'), + ('oracle_fdw'), + ('orafce'), + ('pageinspect'), + ('pg_bigm'), + ('pg_buffercache'), + ('pg_freespacemap'), + ('pg_hint_plan'), + ('pg_partman'), + ('pg_prewarm'), + ('pg_proctab'), + ('pg_similarity'), + ('pg_tle'), + ('pg_transport'), + ('pg_visibility'), + ('pgcrypto'), + ('pgstattuple'), + ('pgvector'), + ('plperl'), + ('plprofiler'), + ('plrust'), + ('pltcl'), + ('prefix'), + ('rdkit'), + ('rds_tools'), + ('tds_fdw'), + ('test_parser'), + ('wal2json'); + +grant insert (partial_name, handle, control_description, control_relocatable, control_requires) + on app.packages + to authenticated; + +grant update (control_description, control_relocatable, control_requires) + on app.packages + to authenticated; + +create or replace function public.publish_package( + package_name app.valid_name, + package_description varchar(1000), + relocatable bool default false, + requires text[] default '{}' +) + returns void + language plpgsql +as $$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); + require text; +begin + if account.handle is null then + raise exception 'user not logged in'; + end if; + + foreach require in array requires + loop + if not exists ( + select true + from app.allowed_extensions + where + name = require + ) then + raise exception '`requires` in the control file can''t have `%` in it', require; + end if; + end loop; + + insert into app.packages(handle, partial_name, control_description, control_relocatable, control_requires) + values (account.handle, package_name, package_description, relocatable, requires) + on conflict on constraint packages_handle_partial_name_key + do update + set control_description = excluded.control_description, + control_relocatable = excluded.control_relocatable, + control_requires = excluded.control_requires; +end; +$$; diff --git a/crates/pgls_pretty_print/tests/data/multi/20231205051816_add_default_version.sql b/crates/pgls_pretty_print/tests/data/multi/20231205051816_add_default_version.sql new file mode 100644 index 000000000..a7f8fc347 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20231205051816_add_default_version.sql @@ -0,0 +1,95 @@ +-- default_version column has a default value '0.0.0' only temporarily because the column is not null. +-- It will be removed below. +alter table app.packages +add column default_version_struct app.semver not null default app.text_to_semver('0.0.0'), +add column default_version text generated always as (app.semver_to_text(default_version_struct)) stored; + +-- for now we set the default version to current latest version +-- new client will allow users to set a specific default version in the control file +update app.packages +set default_version_struct = app.text_to_semver(pp.latest_version) +from public.packages pp +where packages.id = pp.id; + +-- now that every row has a valid default_version, remove the default value of '0.0.0' +alter table app.packages +alter column default_version_struct drop default; + +-- add new default_version column to the view +create or replace view public.packages as + select + pa.id, + pa.package_name, + pa.handle, + pa.partial_name, + newest_ver.version as latest_version, + newest_ver.description_md, + pa.control_description, + pa.control_requires, + pa.created_at, + pa.default_version + from + app.packages pa, + lateral ( + select * + from app.package_versions pv + where pv.package_id = pa.id + order by pv.version_struct desc + limit 1 + ) newest_ver; + +-- grant insert and update permissions to authenticated users on the new default_version_struct column +grant insert (partial_name, handle, control_description, control_relocatable, control_requires, default_version_struct) + on app.packages + to authenticated; + +grant update (control_description, control_relocatable, control_requires, default_version_struct) + on app.packages + to authenticated; + +-- publish_package accepts an additional `default_version` argument +drop function public.publish_package(app.valid_name, varchar, bool, text[]); +create or replace function public.publish_package( + package_name app.valid_name, + package_description varchar(1000), + relocatable bool default false, + requires text[] default '{}', + default_version text default null +) + returns void + language plpgsql +as $$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); + require text; +begin + if account.handle is null then + raise exception 'user not logged in'; + end if; + + if default_version is null then + raise exception 'default_version is required. If you are on `dbdev` CLI version 0.1.5 or older upgrade to the latest version.'; + end if; + + foreach require in array requires + loop + if not exists ( + select true + from app.allowed_extensions + where + name = require + ) then + raise exception '`requires` in the control file can''t have `%` in it', require; + end if; + end loop; + + insert into app.packages(handle, partial_name, control_description, control_relocatable, control_requires, default_version_struct) + values (account.handle, package_name, package_description, relocatable, requires, app.text_to_semver(default_version)) + on conflict on constraint packages_handle_partial_name_key + do update + set control_description = excluded.control_description, + control_relocatable = excluded.control_relocatable, + control_requires = excluded.control_requires, + default_version_struct = excluded.default_version_struct; +end; +$$; diff --git a/crates/pgls_pretty_print/tests/data/multi/20231205101809_dbdev_supports_default_version.sql b/crates/pgls_pretty_print/tests/data/multi/20231205101809_dbdev_supports_default_version.sql new file mode 100644 index 000000000..761bc5679 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20231205101809_dbdev_supports_default_version.sql @@ -0,0 +1,340 @@ +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_name = 'supabase-dbdev'), +(0,0,4), +$pkg$ + +create schema dbdev; + +-- base_url and api_key have been added as arguments with default values to help test locally +create or replace function dbdev.install( + package_name text, + base_url text default 'https://api.database.dev/rest/v1/', + api_key text default 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyYndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzIsImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJrzM0AQKsu_5k134s' +) + returns bool + language plpgsql +as $$ +declare + http_ext_schema regnamespace = extnamespace::regnamespace from pg_catalog.pg_extension where extname = 'http' limit 1; + pgtle_is_available bool = true from pg_catalog.pg_extension where extname = 'pg_tle' limit 1; + -- HTTP respones + rec jsonb; + status int; + contents json; + + -- Install Record + rec_sql text; + rec_ver text; + rec_from_ver text; + rec_to_ver text; + rec_package_name text; + rec_description text; + rec_requires text[]; + rec_default_ver text; +begin + + if http_ext_schema is null then + raise exception using errcode='22000', message=format('dbdev requires the http extension and it is not available'); + end if; + + if pgtle_is_available is null then + raise exception using errcode='22000', message=format('dbdev requires the pgtle extension and it is not available'); + end if; + + ------------------- + -- Base Versions -- + ------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + 'GET', + format( + '%spackage_versions?select=package_name,version,sql,control_description,control_requires&limit=50&package_name=eq.%s', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + ('apiKey', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> 'status')::int; + contents = to_json(rec ->> 'content') #>> '{}'; + + if status <> 200 then + raise notice using errcode='22000', message=format('DBDEV INFO: %s', contents); + raise exception using errcode='22000', message=format('Non-200 response code while loading versions from dbdev'); + end if; + + if contents is null or json_typeof(contents) <> 'array' or json_array_length(contents) = 0 then + raise exception using errcode='22000', message=format('No versions found for package named %s', package_name); + end if; + + for rec_package_name, rec_ver, rec_sql, rec_description, rec_requires in select + (r ->> 'package_name'), + (r ->> 'version'), + (r ->> 'sql'), + (r ->> 'control_description'), + array(select json_array_elements_text((r -> 'control_requires'))) + from + json_array_elements(contents) as r + loop + + -- Install the primary version + if not exists ( + select true + from pgtle.available_extensions() + where + name = rec_package_name + ) then + perform pgtle.install_extension(rec_package_name, rec_ver, rec_description, rec_sql, rec_requires); + end if; + + -- Install other available versions + if not exists ( + select true + from pgtle.available_extension_versions() + where + name = rec_package_name + and version = rec_ver + ) then + perform pgtle.install_extension_version_sql(rec_package_name, rec_ver, rec_sql); + end if; + + end loop; + + ---------------------- + -- Upgrade Versions -- + ---------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + 'GET', + format( + '%spackage_upgrades?select=package_name,from_version,to_version,sql&limit=50&package_name=eq.%s', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + ('apiKey', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> 'status')::int; + contents = to_json(rec ->> 'content') #>> '{}'; + + if status <> 200 then + raise notice using errcode='22000', message=format('DBDEV INFO: %s', contents); + raise exception using errcode='22000', message=format('Non-200 response code while loading upgrade paths from dbdev'); + end if; + + if json_typeof(contents) <> 'array' then + raise exception using errcode='22000', message=format('Invalid response from dbdev upgrade paths'); + end if; + + for rec_package_name, rec_from_ver, rec_to_ver, rec_sql in select + (r ->> 'package_name'), + (r ->> 'from_version'), + (r ->> 'to_version'), + (r ->> 'sql') + from + json_array_elements(contents) as r + loop + + if not exists ( + select true + from pgtle.extension_update_paths(rec_package_name) + where + source = rec_from_ver + and target = rec_to_ver + and path is not null + ) then + perform pgtle.install_update_path(rec_package_name, rec_from_ver, rec_to_ver, rec_sql); + end if; + end loop; + + ------------------------- + -- Set Default Version -- + ------------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + 'GET', + format( + '%spackages?select=package_name,default_version&limit=1&package_name=eq.%s', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + ('apiKey', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> 'status')::int; + contents = to_json(rec ->> 'content') #>> '{}'; + + if status <> 200 then + raise notice using errcode='22000', message=format('DBDEV INFO: %s', contents); + raise exception using errcode='22000', message=format('Non-200 response code while loading packages from dbdev'); + end if; + + if contents is null or json_typeof(contents) <> 'array' or json_array_length(contents) = 0 then + raise exception using errcode='22000', message=format('No package named %s found', package_name); + end if; + + for rec_package_name, rec_default_ver in select + (r ->> 'package_name'), + (r ->> 'default_version') + from + json_array_elements(contents) as r + loop + + if rec_default_ver is not null then + perform pgtle.set_default_version(rec_package_name, rec_default_ver); + else + raise notice using errcode='22000', message=format('DBDEV INFO: missing default version'); + end if; + + end loop; + + -------------------------- + -- Send Download Notice -- + -------------------------- + -- Notifies dbdev that a package has been downloaded and records IP + user agent so we can compute unique download counts + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + 'POST', + format( + '%srpc/register_download', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$ + ), + array[ + ('apiKey', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header, + ('x-client-info', 'dbdev/0.0.4')::http_header + ], + 'application/json', + json_build_object('package_name', $stmt$ || pg_catalog.quote_literal($1) || $stmt$)::text + ) + ) x + limit 1; $stmt$ + into rec; + + return true; +end; +$$; + +$pkg$, +$description$ +# dbdev + +dbdev is the SQL client for database.new and is the primary way end users interact with the package (pglet) registry. + +dbdev can be used to load packages from the registry. For example: + +```sql +-- Load the package from the package index +select dbdev.install('olirice-index_advisor'); +``` +Where `olirice` is the handle of the author and `index_advisor` is the name of the pglet. + +Once installed, pglets are visible in PostgreSQL as extensions. At that point they can be enabled with standard Postgres commands i.e. the `create extension` + +To improve reproducibility, we recommend __always__ specifying the package version in your `create extension` statements. + +For example: +```sql +-- Enable the extension +create extension "olirice-index_advisor" + schema 'public' + version '0.1.0'; +``` + +Which creates all tables/indexes/functions/etc specified by the extension. + +## How to Install + +The in-database SQL client for the package registry is named `dbdev`. You can bootstrap the client with: + +```sql +/*--------------------- +---- install dbdev ---- +---------------------- +Requires: + - pg_tle: https://github.com/aws/pg_tle + - pgsql-http: https://github.com/pramsey/pgsql-http +*/ +create extension if not exists http with schema extensions; +create extension if not exists pg_tle; +drop extension if exists "supabase-dbdev"; +select pgtle.uninstall_extension_if_exists('supabase-dbdev'); +select + pgtle.install_extension( + 'supabase-dbdev', + resp.contents ->> 'version', + 'PostgreSQL package manager', + resp.contents ->> 'sql' + ) +from http( + ( + 'GET', + 'https://api.database.dev/rest/v1/' + || 'package_versions?select=sql,version' + || '&package_name=eq.supabase-dbdev' + || '&order=version.desc' + || '&limit=1', + array[ + ( + 'apiKey', + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJp' + || 'c3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyY' + || 'ndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzI' + || 'sImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJ' + || 'rzM0AQKsu_5k134s' + )::http_header + ], + null, + null + ) +) x, +lateral ( + select + ((row_to_json(x) -> 'content') #>> '{}')::json -> 0 +) resp(contents); +create extension "supabase-dbdev"; +select dbdev.install('supabase-dbdev'); +drop extension if exists "supabase-dbdev"; +create extension "supabase-dbdev"; +``` + +With the client ready, search for packages on [database.dev](database.dev) and install them with + +```sql +select dbdev.install('handle-package_name'); +create extension "handle-package_name" + schema 'public' + version '1.2.3'; +``` +$description$ +); + +-- set supabase-dbdev package's default_version to 0.0.4 +update app.packages +set default_version_struct = app.text_to_semver('0.0.4') +where package_name = 'supabase-dbdev'; diff --git a/crates/pgls_pretty_print/tests/data/multi/20231207071422_new_package_name.sql b/crates/pgls_pretty_print/tests/data/multi/20231207071422_new_package_name.sql new file mode 100644 index 000000000..4c6d51382 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20231207071422_new_package_name.sql @@ -0,0 +1,82 @@ +create or replace function app.to_package_name(handle app.valid_name, partial_name app.valid_name) + returns text + immutable + language sql +as $$ + select format('%s@%s', $1, $2) +$$; + +alter table app.packages +add column package_alias text null; + +update app.packages +set package_alias = format('%s@%s', handle, partial_name); + +-- add package_alias column to the views +create or replace view public.packages as + select + pa.id, + pa.package_name, + pa.handle, + pa.partial_name, + newest_ver.version as latest_version, + newest_ver.description_md, + pa.control_description, + pa.control_requires, + pa.created_at, + pa.default_version, + pa.package_alias + from + app.packages pa, + lateral ( + select * + from app.package_versions pv + where pv.package_id = pa.id + order by pv.version_struct desc + limit 1 + ) newest_ver; + +create or replace view public.package_versions as + select + pv.id, + pv.package_id, + pa.package_name, + pv.version, + pv.sql, + pv.description_md, + pa.control_description, + pa.control_requires, + pv.created_at, + pa.package_alias + from + app.packages pa + join app.package_versions pv + on pa.id = pv.package_id; + +create or replace view public.package_upgrades + as + select + pu.id, + pu.package_id, + pa.package_name, + pu.from_version, + pu.to_version, + pu.sql, + pu.created_at, + pa.package_alias + from + app.packages pa + join app.package_upgrades pu + on pa.id = pu.package_id; + +create or replace function public.register_download(package_name text) + returns void + language sql + security definer + as +$$ + insert into app.downloads(package_id) + select id + from app.packages ap + where ap.package_name = $1 or ap.package_alias = $1 +$$; diff --git a/crates/pgls_pretty_print/tests/data/multi/20231207073048_dbdev_supports_new_package_names.sql b/crates/pgls_pretty_print/tests/data/multi/20231207073048_dbdev_supports_new_package_names.sql new file mode 100644 index 000000000..810c617da --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20231207073048_dbdev_supports_new_package_names.sql @@ -0,0 +1,342 @@ +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_alias= 'supabase@dbdev'), +(0,0,5), +$pkg$ + +create schema dbdev; + +-- base_url and api_key have been added as arguments with default values to help test locally +create or replace function dbdev.install( + package_name text, + base_url text default 'https://api.database.dev/rest/v1/', + api_key text default 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyYndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzIsImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJrzM0AQKsu_5k134s' +) + returns bool + language plpgsql +as $$ +declare + http_ext_schema regnamespace = extnamespace::regnamespace from pg_catalog.pg_extension where extname = 'http' limit 1; + pgtle_is_available bool = true from pg_catalog.pg_extension where extname = 'pg_tle' limit 1; + -- HTTP respones + rec jsonb; + status int; + contents json; + + -- Install Record + rec_sql text; + rec_ver text; + rec_from_ver text; + rec_to_ver text; + rec_description text; + rec_requires text[]; + rec_default_ver text; +begin + + if http_ext_schema is null then + raise exception using errcode='22000', message=format('dbdev requires the http extension and it is not available'); + end if; + + if pgtle_is_available is null then + raise exception using errcode='22000', message=format('dbdev requires the pgtle extension and it is not available'); + end if; + + ------------------- + -- Base Versions -- + ------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + 'GET', + format( + '%spackage_versions?select=version,sql,control_description,control_requires&limit=50&or=(package_name.eq.%s,package_alias.eq.%s)', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$ + ), + array[ + ('apiKey', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> 'status')::int; + contents = to_json(rec ->> 'content') #>> '{}'; + + if status <> 200 then + raise notice using errcode='22000', message=format('DBDEV INFO: %s', contents); + raise exception using errcode='22000', message=format('Non-200 response code while loading versions from dbdev'); + end if; + + if contents is null or json_typeof(contents) <> 'array' or json_array_length(contents) = 0 then + raise exception using errcode='22000', message=format('No versions found for package named %s', package_name); + end if; + + for rec_ver, rec_sql, rec_description, rec_requires in select + (r ->> 'version'), + (r ->> 'sql'), + (r ->> 'control_description'), + array(select json_array_elements_text((r -> 'control_requires'))) + from + json_array_elements(contents) as r + loop + + -- Install the primary version + if not exists ( + select true + from pgtle.available_extensions() + where + name = package_name + ) then + perform pgtle.install_extension(package_name, rec_ver, rec_description, rec_sql, rec_requires); + end if; + + -- Install other available versions + if not exists ( + select true + from pgtle.available_extension_versions() + where + name = package_name + and version = rec_ver + ) then + perform pgtle.install_extension_version_sql(package_name, rec_ver, rec_sql); + end if; + + end loop; + + ---------------------- + -- Upgrade Versions -- + ---------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + 'GET', + format( + '%spackage_upgrades?select=from_version,to_version,sql&limit=50&or=(package_name.eq.%s,package_alias.eq.%s)', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$ + ), + array[ + ('apiKey', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> 'status')::int; + contents = to_json(rec ->> 'content') #>> '{}'; + + if status <> 200 then + raise notice using errcode='22000', message=format('DBDEV INFO: %s', contents); + raise exception using errcode='22000', message=format('Non-200 response code while loading upgrade paths from dbdev'); + end if; + + if json_typeof(contents) <> 'array' then + raise exception using errcode='22000', message=format('Invalid response from dbdev upgrade paths'); + end if; + + for rec_from_ver, rec_to_ver, rec_sql in select + (r ->> 'from_version'), + (r ->> 'to_version'), + (r ->> 'sql') + from + json_array_elements(contents) as r + loop + + if not exists ( + select true + from pgtle.extension_update_paths(package_name) + where + source = rec_from_ver + and target = rec_to_ver + and path is not null + ) then + perform pgtle.install_update_path(package_name, rec_from_ver, rec_to_ver, rec_sql); + end if; + end loop; + + ------------------------- + -- Set Default Version -- + ------------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + 'GET', + format( + '%spackages?select=default_version&limit=1&or=(package_name.eq.%s,package_alias.eq.%s)', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$ + ), + array[ + ('apiKey', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> 'status')::int; + contents = to_json(rec ->> 'content') #>> '{}'; + + if status <> 200 then + raise notice using errcode='22000', message=format('DBDEV INFO: %s', contents); + raise exception using errcode='22000', message=format('Non-200 response code while loading packages from dbdev'); + end if; + + if contents is null or json_typeof(contents) <> 'array' or json_array_length(contents) = 0 then + raise exception using errcode='22000', message=format('No package named %s found', package_name); + end if; + + for rec_default_ver in select + (r ->> 'default_version') + from + json_array_elements(contents) as r + loop + + if rec_default_ver is not null then + perform pgtle.set_default_version(package_name, rec_default_ver); + else + raise notice using errcode='22000', message=format('DBDEV INFO: missing default version'); + end if; + + end loop; + + -------------------------- + -- Send Download Notice -- + -------------------------- + -- Notifies dbdev that a package has been downloaded and records IP + user agent so we can compute unique download counts + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + 'POST', + format( + '%srpc/register_download', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$ + ), + array[ + ('apiKey', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header, + ('x-client-info', 'dbdev/0.0.5')::http_header + ], + 'application/json', + json_build_object('package_name', $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$)::text + ) + ) x + limit 1; $stmt$ + into rec; + + return true; +end; +$$; + +$pkg$, +$description$ +# dbdev + +dbdev is the SQL client for database.new and is the primary way end users interact with the package registry. + +dbdev can be used to load packages from the registry. For example: + +```sql +-- Load the package from the package index +select dbdev.install('olirice@index_advisor'); +``` +Where `olirice` is the handle of the author and `index_advisor` is the name of the package. + +Once installed, packages are visible in PostgreSQL as extensions. At that point they can be enabled with standard Postgres commands i.e. the `create extension` + +To improve reproducibility, we recommend __always__ specifying the package version in your `create extension` statements. + +For example: +```sql +-- Enable the extension +create extension "olirice@index_advisor" + schema 'public' + version '0.1.0'; +``` + +Which creates all tables/indexes/functions/etc specified by the extension. + +## How to Install + +The in-database SQL client for the package registry is named `dbdev`. You can bootstrap the client with: + +```sql +/*--------------------- +---- install dbdev ---- +---------------------- +Requires: + - pg_tle: https://github.com/aws/pg_tle + - pgsql-http: https://github.com/pramsey/pgsql-http +*/ +create extension if not exists http with schema extensions; +create extension if not exists pg_tle; +-- drop dbdev with older naming scheme if present +drop extension if exists "supabase-dbdev"; +select pgtle.uninstall_extension_if_exists('supabase-dbdev'); +drop extension if exists "supabase@dbdev"; +select pgtle.uninstall_extension_if_exists('supabase@dbdev'); +select + pgtle.install_extension( + 'supabase@dbdev', + resp.contents ->> 'version', + 'PostgreSQL package manager', + resp.contents ->> 'sql' + ) +from http( + ( + 'GET', + 'https://api.database.dev/rest/v1/' + || 'package_versions?select=sql,version' + || '&package_alias=eq.supabase@dbdev' + || '&order=version.desc' + || '&limit=1', + array[ + ( + 'apiKey', + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJp' + || 'c3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyY' + || 'ndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzI' + || 'sImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJ' + || 'rzM0AQKsu_5k134s' + )::http_header + ], + null, + null + ) +) x, +lateral ( + select + ((row_to_json(x) -> 'content') #>> '{}')::json -> 0 +) resp(contents); +create extension "supabase@dbdev"; +select dbdev.install('supabase@dbdev'); +drop extension if exists "supabase@dbdev"; +create extension "supabase@dbdev"; +``` + +With the client ready, search for packages on [database.dev](database.dev) and install them with + +```sql +select dbdev.install('handle@package_name'); +create extension "handle@package_name" + schema 'public' + version '1.2.3'; +``` +$description$ +); + +-- set supabase@dbdev package's default_version to 0.0.5 +update app.packages +set default_version_struct = app.text_to_semver('0.0.5') +where package_alias = 'supabase@dbdev'; diff --git a/crates/pgls_pretty_print/tests/data/multi/20231207111703_langchain@embedding_search-1.1.1.sql b/crates/pgls_pretty_print/tests/data/multi/20231207111703_langchain@embedding_search-1.1.1.sql new file mode 100644 index 000000000..88c659c30 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20231207111703_langchain@embedding_search-1.1.1.sql @@ -0,0 +1,167 @@ +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_alias= 'langchain@embedding_search'), +(1,1,1), +$pkg$ +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = 'vector' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception '"langchain@embedding_search" requires "vector"' + using hint = 'Run "create extension vector" and try again'; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int, + filter jsonb DEFAULT '{}' +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + where metadata @> filter + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +$pkg$, + +$description_md$ +# embedding_search + +[LangChain](https://js.langchain.com/docs/) is a framework for developing applications powered by language models with a plugable architecture. + +`langchain@embedding_search` uses a Supabase Postgres database as its vector store. + +## Installation + +```sql +select dbdev.install('langchain@embedding_search'); +create extension if not exists vector; +create extension "langchain@embedding_search" + schema public + version '1.1.0'; +``` +Note: + +`vector` is a dependency of `langchain@embedding_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +### Standard Usage + +The below example shows how to perform a basic similarity search with Supabase: + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromTexts( + ["Hello world", "Bye bye", "What's this?"], + [{ id: 2 }, { id: 1 }, { id: 3 }], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + const resultOne = await vectorStore.similaritySearch("Hello world", 1); + + console.log(resultOne); +}; +``` + +### Metadata Filtering + +Given the above `match_documents` Postgres function, you can also pass a filter parameter to only documents with a specific metadata field value. + +```js +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; + +// First, follow set-up instructions at +// https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromTexts( + ["Hello world", "Hello world", "Hello world"], + [{ user_id: 2 }, { user_id: 1 }, { user_id: 3 }], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + const result = await vectorStore.similaritySearch("Hello world", 1, { + user_id: 3, + }); + + console.log(result); +}; +``` + +For more details, checkout the LangChain Supabase integration docs: https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase +$description_md$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20231207112129_langchain@hybrid_search-1.1.1.sql b/crates/pgls_pretty_print/tests/data/multi/20231207112129_langchain@hybrid_search-1.1.1.sql new file mode 100644 index 000000000..e5f65f9ee --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20231207112129_langchain@hybrid_search-1.1.1.sql @@ -0,0 +1,144 @@ +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_alias = 'langchain@hybrid_search'), +(1,1,1), +$pkg$ +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = 'vector' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception '"langchain@hybrid_search" requires "vector"' + using hint = 'Run "create extension vector" and try again'; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to similarity search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int, + filter jsonb DEFAULT '{}' +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + where metadata @> filter + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +-- Create a function to keyword search for documents +create function kw_match_documents(query_text text, match_count int) +returns table (id bigint, content text, metadata jsonb, similarity real) +as $$ + +begin +return query execute +format(' + select + id, content, metadata, ts_rank(to_tsvector(content), plainto_tsquery($1)) as similarity + from + documents + where + to_tsvector(content) @@ plainto_tsquery($1) + order by + similarity desc + limit $2 +') +using query_text, match_count; +end; +$$ language plpgsql; + +$pkg$, + +$description_md$ +# hybrid_search + +Langchain supports hybrid search with a Supabase Postgres database. The hybrid search combines the postgres pgvector extension (similarity search) and Full-Text Search (keyword search) to retrieve documents. You can add documents via SupabaseVectorStore addDocuments function. SupabaseHybridKeyWordSearch accepts embedding, supabase client, number of results for similarity search, and number of results for keyword search as parameters. The getRelevantDocuments function produces a list of documents that has duplicates removed and is sorted by relevance score. + +## Installation + +```sql +select dbdev.install('langchain@hybrid_search'); +create extension if not exists vector; +create extension "langchain@hybrid_search" + schema public + version '1.1.0'; +``` +Note: + +`vector` is a dependency of `langchain@hybrid_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; +import { SupabaseHybridSearch } from "langchain/retrievers/supabase"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const embeddings = new OpenAIEmbeddings(); + + const retriever = new SupabaseHybridSearch(embeddings, { + client, + // Below are the defaults, expecting that you set up your supabase table and functions according to the guide above. Please change if necessary. + similarityK: 2, + keywordK: 2, + tableName: "documents", + similarityQueryName: "match_documents", + keywordQueryName: "kw_match_documents", + }); + + const results = await retriever.getRelevantDocuments("hello bye"); + + console.log(results); +}; +``` + +For more details, checkout the LangChain Supabase Hybrid Search docs: https://js.langchain.com/docs/modules/indexes/retrievers/supabase-hybrid +$description_md$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20231207112942_michelp@adminpack-0.0.2.sql b/crates/pgls_pretty_print/tests/data/multi/20231207112942_michelp@adminpack-0.0.2.sql new file mode 100644 index 000000000..92a712d0c --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20231207112942_michelp@adminpack-0.0.2.sql @@ -0,0 +1,1555 @@ +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_alias = 'michelp@adminpack'), +(0,0,2), +$adminpack$ +-- From: https://github.com/ioguix/pgsql-bloat-estimation + +-- Copyright (c) 2015-2019, Jehan-Guillaume (ioguix) de Rorthais +-- All rights reserved. + +-- Redistribution and use in source and binary forms, with or without +-- modification, are permitted provided that the following conditions are met: + +-- * Redistributions of source code must retain the above copyright notice, this +-- list of conditions and the following disclaimer. + +-- * Redistributions in binary form must reproduce the above copyright notice, +-- this list of conditions and the following disclaimer in the documentation +-- and/or other materials provided with the distribution. + +-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +-- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +-- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +-- DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +-- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +-- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +-- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +-- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +-- OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +-- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-- WARNING: executed with a non-superuser role, the query inspect only index on tables you are granted to read. +-- WARNING: rows with is_na = 't' are known to have bad statistics ("name" type is not supported). +-- This query is compatible with PostgreSQL 8.2 and after +CREATE VIEW index_bloat AS SELECT current_database(), nspname AS schemaname, tblname, idxname, bs*(relpages)::bigint AS real_size, + bs*(relpages-est_pages)::bigint AS extra_size, + 100 * (relpages-est_pages)::float / relpages AS extra_pct, + fillfactor, + CASE WHEN relpages > est_pages_ff + THEN bs*(relpages-est_pages_ff) + ELSE 0 + END AS bloat_size, + 100 * (relpages-est_pages_ff)::float / relpages AS bloat_pct, + is_na + -- , 100-(pst).avg_leaf_density AS pst_avg_bloat, est_pages, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples, relpages -- (DEBUG INFO) +FROM ( + SELECT coalesce(1 + + ceil(reltuples/floor((bs-pageopqdata-pagehdr)/(4+nulldatahdrwidth)::float)), 0 -- ItemIdData size + computed avg size of a tuple (nulldatahdrwidth) + ) AS est_pages, + coalesce(1 + + ceil(reltuples/floor((bs-pageopqdata-pagehdr)*fillfactor/(100*(4+nulldatahdrwidth)::float))), 0 + ) AS est_pages_ff, + bs, nspname, tblname, idxname, relpages, fillfactor, is_na + -- , pgstatindex(idxoid) AS pst, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples -- (DEBUG INFO) + FROM ( + SELECT maxalign, bs, nspname, tblname, idxname, reltuples, relpages, idxoid, fillfactor, + ( index_tuple_hdr_bm + + maxalign - CASE -- Add padding to the index tuple header to align on MAXALIGN + WHEN index_tuple_hdr_bm%maxalign = 0 THEN maxalign + ELSE index_tuple_hdr_bm%maxalign + END + + nulldatawidth + maxalign - CASE -- Add padding to the data to align on MAXALIGN + WHEN nulldatawidth = 0 THEN 0 + WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign + ELSE nulldatawidth::integer%maxalign + END + )::numeric AS nulldatahdrwidth, pagehdr, pageopqdata, is_na + -- , index_tuple_hdr_bm, nulldatawidth -- (DEBUG INFO) + FROM ( + SELECT n.nspname, i.tblname, i.idxname, i.reltuples, i.relpages, + i.idxoid, i.fillfactor, current_setting('block_size')::numeric AS bs, + CASE -- MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?) + WHEN version() ~ 'mingw32' OR version() ~ '64-bit|x86_64|ppc64|ia64|amd64' THEN 8 + ELSE 4 + END AS maxalign, + /* per page header, fixed size: 20 for 7.X, 24 for others */ + 24 AS pagehdr, + /* per page btree opaque data */ + 16 AS pageopqdata, + /* per tuple header: add IndexAttributeBitMapData if some cols are null-able */ + CASE WHEN max(coalesce(s.null_frac,0)) = 0 + THEN 8 -- IndexTupleData size + ELSE 8 + (( 32 + 8 - 1 ) / 8) -- IndexTupleData size + IndexAttributeBitMapData size ( max num filed per index + 8 - 1 /8) + END AS index_tuple_hdr_bm, + /* data len: we remove null values save space using it fractionnal part from stats */ + sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) AS nulldatawidth, + max( CASE WHEN i.atttypid = 'pg_catalog.name'::regtype THEN 1 ELSE 0 END ) > 0 AS is_na + FROM ( + SELECT ct.relname AS tblname, ct.relnamespace, ic.idxname, ic.attpos, ic.indkey, ic.indkey[ic.attpos], ic.reltuples, ic.relpages, ic.tbloid, ic.idxoid, ic.fillfactor, + coalesce(a1.attnum, a2.attnum) AS attnum, coalesce(a1.attname, a2.attname) AS attname, coalesce(a1.atttypid, a2.atttypid) AS atttypid, + CASE WHEN a1.attnum IS NULL + THEN ic.idxname + ELSE ct.relname + END AS attrelname + FROM ( + SELECT idxname, reltuples, relpages, tbloid, idxoid, fillfactor, indkey, + pg_catalog.generate_series(1,indnatts) AS attpos + FROM ( + SELECT ci.relname AS idxname, ci.reltuples, ci.relpages, i.indrelid AS tbloid, + i.indexrelid AS idxoid, + coalesce(substring( + array_to_string(ci.reloptions, ' ') + from 'fillfactor=([0-9]+)')::smallint, 90) AS fillfactor, + i.indnatts, + pg_catalog.string_to_array(pg_catalog.textin( + pg_catalog.int2vectorout(i.indkey)),' ')::int[] AS indkey + FROM pg_catalog.pg_index i + JOIN pg_catalog.pg_class ci ON ci.oid = i.indexrelid + WHERE ci.relam=(SELECT oid FROM pg_am WHERE amname = 'btree') + AND ci.relpages > 0 + ) AS idx_data + ) AS ic + JOIN pg_catalog.pg_class ct ON ct.oid = ic.tbloid + LEFT JOIN pg_catalog.pg_attribute a1 ON + ic.indkey[ic.attpos] <> 0 + AND a1.attrelid = ic.tbloid + AND a1.attnum = ic.indkey[ic.attpos] + LEFT JOIN pg_catalog.pg_attribute a2 ON + ic.indkey[ic.attpos] = 0 + AND a2.attrelid = ic.idxoid + AND a2.attnum = ic.attpos + ) i + JOIN pg_catalog.pg_namespace n ON n.oid = i.relnamespace + JOIN pg_catalog.pg_stats s ON s.schemaname = n.nspname + AND s.tablename = i.attrelname + AND s.attname = i.attname + GROUP BY 1,2,3,4,5,6,7,8,9,10,11 + ) AS rows_data_stats + ) AS rows_hdr_pdg_stats +) AS relation_stats +ORDER BY nspname, tblname, idxname; + +CREATE VIEW table_bloat AS /* WARNING: executed with a non-superuser role, the query inspect only tables and materialized view (9.3+) you are granted to read. +* This query is compatible with PostgreSQL 9.0 and more +*/ +SELECT current_database(), schemaname, tblname, bs*tblpages AS real_size, + (tblpages-est_tblpages)*bs AS extra_size, + CASE WHEN tblpages > 0 AND tblpages - est_tblpages > 0 + THEN 100 * (tblpages - est_tblpages)/tblpages::float + ELSE 0 + END AS extra_pct, fillfactor, + CASE WHEN tblpages - est_tblpages_ff > 0 + THEN (tblpages-est_tblpages_ff)*bs + ELSE 0 + END AS bloat_size, + CASE WHEN tblpages > 0 AND tblpages - est_tblpages_ff > 0 + THEN 100 * (tblpages - est_tblpages_ff)/tblpages::float + ELSE 0 + END AS bloat_pct, is_na + -- , tpl_hdr_size, tpl_data_size, (pst).free_percent + (pst).dead_tuple_percent AS real_frag -- (DEBUG INFO) +FROM ( + SELECT ceil( reltuples / ( (bs-page_hdr)/tpl_size ) ) + ceil( toasttuples / 4 ) AS est_tblpages, + ceil( reltuples / ( (bs-page_hdr)*fillfactor/(tpl_size*100) ) ) + ceil( toasttuples / 4 ) AS est_tblpages_ff, + tblpages, fillfactor, bs, tblid, schemaname, tblname, heappages, toastpages, is_na + -- , tpl_hdr_size, tpl_data_size, pgstattuple(tblid) AS pst -- (DEBUG INFO) + FROM ( + SELECT + ( 4 + tpl_hdr_size + tpl_data_size + (2*ma) + - CASE WHEN tpl_hdr_size%ma = 0 THEN ma ELSE tpl_hdr_size%ma END + - CASE WHEN ceil(tpl_data_size)::int%ma = 0 THEN ma ELSE ceil(tpl_data_size)::int%ma END + ) AS tpl_size, bs - page_hdr AS size_per_block, (heappages + toastpages) AS tblpages, heappages, + toastpages, reltuples, toasttuples, bs, page_hdr, tblid, schemaname, tblname, fillfactor, is_na + -- , tpl_hdr_size, tpl_data_size + FROM ( + SELECT + tbl.oid AS tblid, ns.nspname AS schemaname, tbl.relname AS tblname, tbl.reltuples, + tbl.relpages AS heappages, coalesce(toast.relpages, 0) AS toastpages, + coalesce(toast.reltuples, 0) AS toasttuples, + coalesce(substring( + array_to_string(tbl.reloptions, ' ') + FROM 'fillfactor=([0-9]+)')::smallint, 100) AS fillfactor, + current_setting('block_size')::numeric AS bs, + CASE WHEN version()~'mingw32' OR version()~'64-bit|x86_64|ppc64|ia64|amd64' THEN 8 ELSE 4 END AS ma, + 24 AS page_hdr, + 23 + CASE WHEN MAX(coalesce(s.null_frac,0)) > 0 THEN ( 7 + count(s.attname) ) / 8 ELSE 0::int END + + CASE WHEN bool_or(att.attname = 'oid' and att.attnum < 0) THEN 4 ELSE 0 END AS tpl_hdr_size, + sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 0) ) AS tpl_data_size, + bool_or(att.atttypid = 'pg_catalog.name'::regtype) + OR sum(CASE WHEN att.attnum > 0 THEN 1 ELSE 0 END) <> count(s.attname) AS is_na + FROM pg_attribute AS att + JOIN pg_class AS tbl ON att.attrelid = tbl.oid + JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace + LEFT JOIN pg_stats AS s ON s.schemaname=ns.nspname + AND s.tablename = tbl.relname AND s.inherited=false AND s.attname=att.attname + LEFT JOIN pg_class AS toast ON tbl.reltoastrelid = toast.oid + WHERE NOT att.attisdropped + AND tbl.relkind in ('r','m') + GROUP BY 1,2,3,4,5,6,7,8,9,10 + ORDER BY 2,3 + ) AS s + ) AS s2 +) AS s3 +-- WHERE NOT is_na +-- AND tblpages*((pst).free_percent + (pst).dead_tuple_percent)::float4/100 >= 1 +ORDER BY schemaname, tblname; + +-- From https://wiki.postgresql.org/wiki/Lock_dependency_information + +CREATE OR REPLACE VIEW blocking_pid_tree AS +WITH RECURSIVE + lock_composite(requested, current) AS (VALUES + ('AccessShareLock'::text, 'AccessExclusiveLock'::text), + ('RowShareLock'::text, 'ExclusiveLock'::text), + ('RowShareLock'::text, 'AccessExclusiveLock'::text), + ('RowExclusiveLock'::text, 'ShareLock'::text), + ('RowExclusiveLock'::text, 'ShareRowExclusiveLock'::text), + ('RowExclusiveLock'::text, 'ExclusiveLock'::text), + ('RowExclusiveLock'::text, 'AccessExclusiveLock'::text), + ('ShareUpdateExclusiveLock'::text, 'ShareUpdateExclusiveLock'::text), + ('ShareUpdateExclusiveLock'::text, 'ShareLock'::text), + ('ShareUpdateExclusiveLock'::text, 'ShareRowExclusiveLock'::text), + ('ShareUpdateExclusiveLock'::text, 'ExclusiveLock'::text), + ('ShareUpdateExclusiveLock'::text, 'AccessExclusiveLock'::text), + ('ShareLock'::text, 'RowExclusiveLock'::text), + ('ShareLock'::text, 'ShareUpdateExclusiveLock'::text), + ('ShareLock'::text, 'ShareRowExclusiveLock'::text), + ('ShareLock'::text, 'ExclusiveLock'::text), + ('ShareLock'::text, 'AccessExclusiveLock'::text), + ('ShareRowExclusiveLock'::text, 'RowExclusiveLock'::text), + ('ShareRowExclusiveLock'::text, 'ShareUpdateExclusiveLock'::text), + ('ShareRowExclusiveLock'::text, 'ShareLock'::text), + ('ShareRowExclusiveLock'::text, 'ShareRowExclusiveLock'::text), + ('ShareRowExclusiveLock'::text, 'ExclusiveLock'::text), + ('ShareRowExclusiveLock'::text, 'AccessExclusiveLock'::text), + ('ExclusiveLock'::text, 'RowShareLock'::text), + ('ExclusiveLock'::text, 'RowExclusiveLock'::text), + ('ExclusiveLock'::text, 'ShareUpdateExclusiveLock'::text), + ('ExclusiveLock'::text, 'ShareLock'::text), + ('ExclusiveLock'::text, 'ShareRowExclusiveLock'::text), + ('ExclusiveLock'::text, 'ExclusiveLock'::text), + ('ExclusiveLock'::text, 'AccessExclusiveLock'::text), + ('AccessExclusiveLock'::text, 'AccessShareLock'::text), + ('AccessExclusiveLock'::text, 'RowShareLock'::text), + ('AccessExclusiveLock'::text, 'RowExclusiveLock'::text), + ('AccessExclusiveLock'::text, 'ShareUpdateExclusiveLock'::text), + ('AccessExclusiveLock'::text, 'ShareLock'::text), + ('AccessExclusiveLock'::text, 'ShareRowExclusiveLock'::text), + ('AccessExclusiveLock'::text, 'ExclusiveLock'::text), + ('AccessExclusiveLock'::text, 'AccessExclusiveLock'::text) + ) +, lock AS ( + SELECT pid, + virtualtransaction, + granted, + mode, + (locktype, + CASE locktype + WHEN 'relation' THEN concat_ws(';', 'db:'||datname, 'rel:'||relation::regclass::text) + WHEN 'extend' THEN concat_ws(';', 'db:'||datname, 'rel:'||relation::regclass::text) + WHEN 'page' THEN concat_ws(';', 'db:'||datname, 'rel:'||relation::regclass::text, 'page#'||page::text) + WHEN 'tuple' THEN concat_ws(';', 'db:'||datname, 'rel:'||relation::regclass::text, 'page#'||page::text, 'tuple#'||tuple::text) + WHEN 'transactionid' THEN transactionid::text + WHEN 'virtualxid' THEN virtualxid::text + WHEN 'object' THEN concat_ws(';', 'class:'||classid::regclass::text, 'objid:'||objid, 'col#'||objsubid) + ELSE concat('db:'||datname) + END::text) AS target + FROM pg_catalog.pg_locks + LEFT JOIN pg_catalog.pg_database ON (pg_database.oid = pg_locks.database) + ) +, waiting_lock AS ( + SELECT + blocker.pid AS blocker_pid, + blocked.pid AS pid, + concat(blocked.mode,blocked.target) AS lock_target + FROM lock blocker + JOIN lock blocked + ON ( NOT blocked.granted + AND blocker.granted + AND blocked.pid != blocker.pid + AND blocked.target IS NOT DISTINCT FROM blocker.target) + JOIN lock_composite c ON (c.requested = blocked.mode AND c.current = blocker.mode) + ) +, acquired_lock AS ( + WITH waiting AS ( + SELECT lock_target, count(lock_target) AS wait_count FROM waiting_lock GROUP BY lock_target + ) + SELECT + pid, + array_agg(concat(mode,target,' + '||wait_count) ORDER BY wait_count DESC NULLS LAST) AS locks_acquired + FROM lock + LEFT JOIN waiting ON waiting.lock_target = concat(mode,target) + WHERE granted + GROUP BY pid + ) +, blocking_lock AS ( + SELECT + ARRAY[date_part('epoch', query_start)::int, pid] AS seq, + 0::int AS depth, + -1::int AS blocker_pid, + pid, + concat('Connect: ',usename,' ',datname,' ',coalesce(host(client_addr)||':'||client_port, 'local') + , E'\nSQL: ',replace(substr(coalesce(query,'N/A'), 1, 60), E'\n', ' ') + , E'\nAcquired:\n ' + , array_to_string(locks_acquired[1:5] || + CASE WHEN array_upper(locks_acquired,1) > 5 + THEN '... '||(array_upper(locks_acquired,1) - 5)::text||' more ...' + END, + E'\n ') + ) AS lock_info, + concat(to_char(query_start, CASE WHEN age(query_start) > '24h' THEN 'Day DD Mon' ELSE 'HH24:MI:SS' END),E' started\n' + ,CASE WHEN wait_event IS NOT NULL THEN 'waiting' ELSE state END,E'\n' + ,date_trunc('second',age(now(),query_start)),' ago' + ) AS lock_state + FROM acquired_lock blocker + LEFT JOIN pg_stat_activity act USING (pid) + WHERE EXISTS + (SELECT 'x' FROM waiting_lock blocked WHERE blocked.blocker_pid = blocker.pid) + AND NOT EXISTS + (SELECT 'x' FROM waiting_lock blocked WHERE blocked.pid = blocker.pid) +UNION ALL + SELECT + blocker.seq || blocked.pid, + blocker.depth + 1, + blocker.pid, + blocked.pid, + concat('Connect: ',usename,' ',datname,' ',coalesce(host(client_addr)||':'||client_port, 'local') + , E'\nSQL: ',replace(substr(coalesce(query,'N/A'), 1, 60), E'\n', ' ') + , E'\nWaiting: ',blocked.lock_target + , CASE WHEN locks_acquired IS NOT NULL + THEN E'\nAcquired:\n ' || + array_to_string(locks_acquired[1:5] || + CASE WHEN array_upper(locks_acquired,1) > 5 + THEN '... '||(array_upper(locks_acquired,1) - 5)::text||' more ...' + END, + E'\n ') + END + ) AS lock_info, + concat(to_char(query_start, CASE WHEN age(query_start) > '24h' THEN 'Day DD Mon' ELSE 'HH24:MI:SS' END),E' started\n' + ,CASE WHEN wait_event IS NOT NULL THEN 'waiting' ELSE state END,E'\n' + ,date_trunc('second',age(now(),query_start)),' ago' + ) AS lock_state + FROM blocking_lock blocker + JOIN waiting_lock blocked + ON (blocked.blocker_pid = blocker.pid) + LEFT JOIN pg_stat_activity act ON (act.pid = blocked.pid) + LEFT JOIN acquired_lock acq ON (acq.pid = blocked.pid) + WHERE blocker.depth < 5 + ) +SELECT concat(lpad('=> ', 4*depth, ' '),pid::text) AS "PID" +, lock_info AS "Lock Info" +, lock_state AS "State" +FROM blocking_lock +ORDER BY seq; + + +-- From https://wiki.postgresql.org/wiki/Index_Maintenance + +CREATE VIEW duplicate_indexes AS SELECT pg_size_pretty(sum(pg_relation_size(idx))::bigint) as size, + (array_agg(idx))[1] as idx1, (array_agg(idx))[2] as idx2, + (array_agg(idx))[3] as idx3, (array_agg(idx))[4] as idx4 +FROM ( + SELECT indexrelid::regclass as idx, (indrelid::text ||E'\n'|| indclass::text ||E'\n'|| indkey::text ||E'\n'|| + coalesce(indexprs::text,'')||E'\n' || coalesce(indpred::text,'')) as key + FROM pg_index) sub +GROUP BY key HAVING count(*)>1 +ORDER BY sum(pg_relation_size(idx)) DESC; + +-- From https://wiki.postgresql.org/wiki/Disk_Usage + +CREATE VIEW table_sizes AS WITH RECURSIVE pg_inherit(inhrelid, inhparent) AS + (select inhrelid, inhparent + FROM pg_inherits + UNION + SELECT child.inhrelid, parent.inhparent + FROM pg_inherit child, pg_inherits parent + WHERE child.inhparent = parent.inhrelid), +pg_inherit_short AS (SELECT * FROM pg_inherit WHERE inhparent NOT IN (SELECT inhrelid FROM pg_inherit)) +SELECT table_schema + , TABLE_NAME + , row_estimate + , pg_size_pretty(total_bytes) AS total + , pg_size_pretty(index_bytes) AS INDEX + , pg_size_pretty(toast_bytes) AS toast + , pg_size_pretty(table_bytes) AS TABLE + , total_bytes::float8 / sum(total_bytes) OVER () AS total_size_share + FROM ( + SELECT *, total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes + FROM ( + SELECT c.oid + , nspname AS table_schema + , relname AS TABLE_NAME + , SUM(c.reltuples) OVER (partition BY parent) AS row_estimate + , SUM(pg_total_relation_size(c.oid)) OVER (partition BY parent) AS total_bytes + , SUM(pg_indexes_size(c.oid)) OVER (partition BY parent) AS index_bytes + , SUM(pg_total_relation_size(reltoastrelid)) OVER (partition BY parent) AS toast_bytes + , parent + FROM ( + SELECT pg_class.oid + , reltuples + , relname + , relnamespace + , pg_class.reltoastrelid + , COALESCE(inhparent, pg_class.oid) parent + FROM pg_class + LEFT JOIN pg_inherit_short ON inhrelid = oid + WHERE relkind IN ('r', 'p') + ) c + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace + ) a + WHERE oid = parent +) a +ORDER BY total_bytes DESC; + +-- From: https://wiki.postgresql.org/wiki/Index_Maintenance + +CREATE VIEW index_usage AS SELECT + t.schemaname, + t.tablename, + c.reltuples::bigint AS num_rows, + pg_size_pretty(pg_relation_size(c.oid)) AS table_size, + psai.indexrelname AS index_name, + pg_size_pretty(pg_relation_size(i.indexrelid)) AS index_size, + CASE WHEN i.indisunique THEN 'Y' ELSE 'N' END AS "unique", + psai.idx_scan AS number_of_scans, + psai.idx_tup_read AS tuples_read, + psai.idx_tup_fetch AS tuples_fetched +FROM + pg_tables t + LEFT JOIN pg_class c ON t.tablename = c.relname + LEFT JOIN pg_index i ON c.oid = i.indrelid + LEFT JOIN pg_stat_all_indexes psai ON i.indexrelid = psai.indexrelid +WHERE + t.schemaname NOT IN ('pg_catalog', 'information_schema') +ORDER BY 1, 2; + +-- From: https://blog.devgenius.io/top-useful-sql-queries-for-postgresql-35ff3355d265 + +CREATE VIEW database_sizes AS SELECT pg_database.datname, + pg_size_pretty(pg_database_size(pg_database.datname)) AS size +FROM pg_database +ORDER BY pg_database_size(pg_database.datname) DESC; + +CREATE VIEW schema_sizes AS SELECT A.schemaname, + pg_size_pretty (SUM(pg_relation_size(C.oid))) as table, + pg_size_pretty (SUM(pg_total_relation_size(C.oid)-pg_relation_size(C.oid))) as index, + pg_size_pretty (SUM(pg_total_relation_size(C.oid))) as table_index, + SUM(n_live_tup) +FROM pg_class C +LEFT JOIN pg_namespace N ON (N.oid = C .relnamespace) +INNER JOIN pg_stat_user_tables A ON C.relname = A.relname +WHERE nspname NOT IN ('pg_catalog', 'information_schema') +AND C .relkind <> 'i' +AND nspname !~ '^pg_toast' +GROUP BY A.schemaname; + +CREATE VIEW last_vacuum_analyze AS SELECT relname, + last_vacuum, + last_autovacuum + n_mod_since_analyze, + last_analyze, + last_autoanalyze, + analyze_count, + autoanalyze_count + FROM pg_stat_user_tables; + +CREATE VIEW table_row_estimates AS SELECT + schemaname, + relname, + n_live_tup +FROM + pg_stat_user_tables +ORDER BY + n_live_tup DESC; + +-- postgres-meta queries as views from: https://github.com/supabase/postgres-meta + +CREATE VIEW pgmeta_columns AS SELECT + c.oid :: int8 AS table_id, + nc.nspname AS schema, + c.relname AS table, + (c.oid || '.' || a.attnum) AS id, + a.attnum AS ordinal_position, + a.attname AS name, + CASE + WHEN a.atthasdef THEN pg_get_expr(ad.adbin, ad.adrelid) + ELSE NULL + END AS default_value, + CASE + WHEN t.typtype = 'd' THEN CASE + WHEN bt.typelem <> 0 :: oid + AND bt.typlen = -1 THEN 'ARRAY' + WHEN nbt.nspname = 'pg_catalog' THEN format_type(t.typbasetype, NULL) + ELSE 'USER-DEFINED' + END + ELSE CASE + WHEN t.typelem <> 0 :: oid + AND t.typlen = -1 THEN 'ARRAY' + WHEN nt.nspname = 'pg_catalog' THEN format_type(a.atttypid, NULL) + ELSE 'USER-DEFINED' + END + END AS data_type, + COALESCE(bt.typname, t.typname) AS format, + a.attidentity IN ('a', 'd') AS is_identity, + CASE + a.attidentity + WHEN 'a' THEN 'ALWAYS' + WHEN 'd' THEN 'BY DEFAULT' + ELSE NULL + END AS identity_generation, + a.attgenerated IN ('s') AS is_generated, + NOT ( + a.attnotnull + OR t.typtype = 'd' AND t.typnotnull + ) AS is_nullable, + ( + c.relkind IN ('r', 'p') + OR c.relkind IN ('v', 'f') AND pg_column_is_updatable(c.oid, a.attnum, FALSE) + ) AS is_updatable, + uniques.table_id IS NOT NULL AS is_unique, + array_to_json( + array( + SELECT + enumlabel + FROM + pg_catalog.pg_enum enums + WHERE + enums.enumtypid = coalesce(bt.oid, t.oid) + OR enums.enumtypid = coalesce(bt.typelem, t.typelem) + ORDER BY + enums.enumsortorder + ) + ) AS enums, + col_description(c.oid, a.attnum) AS comment +FROM + pg_attribute a + LEFT JOIN pg_attrdef ad ON a.attrelid = ad.adrelid + AND a.attnum = ad.adnum + JOIN ( + pg_class c + JOIN pg_namespace nc ON c.relnamespace = nc.oid + ) ON a.attrelid = c.oid + JOIN ( + pg_type t + JOIN pg_namespace nt ON t.typnamespace = nt.oid + ) ON a.atttypid = t.oid + LEFT JOIN ( + pg_type bt + JOIN pg_namespace nbt ON bt.typnamespace = nbt.oid + ) ON t.typtype = 'd' + AND t.typbasetype = bt.oid + LEFT JOIN ( + SELECT + conrelid AS table_id, + conkey[1] AS ordinal_position + FROM pg_catalog.pg_constraint + WHERE contype = 'u' AND cardinality(conkey) = 1 + ) AS uniques ON uniques.table_id = c.oid AND uniques.ordinal_position = a.attnum +WHERE + NOT pg_is_other_temp_schema(nc.oid) + AND a.attnum > 0 + AND NOT a.attisdropped + AND (c.relkind IN ('r', 'v', 'm', 'f', 'p')) + AND ( + pg_has_role(c.relowner, 'USAGE') + OR has_column_privilege( + c.oid, + a.attnum, + 'SELECT, INSERT, UPDATE, REFERENCES' + ) + ); + +CREATE VIEW pgmeta_config AS SELECT + name, + setting, + category, + TRIM(split_part(category, '/', 1)) AS group, + TRIM(split_part(category, '/', 2)) AS subgroup, + unit, + short_desc, + extra_desc, + context, + vartype, + source, + min_val, + max_val, + enumvals, + boot_val, + reset_val, + sourcefile, + sourceline, + pending_restart +FROM + pg_settings +ORDER BY + category, + name; + +CREATE VIEW pgmeta_extensions AS SELECT + e.name, + n.nspname AS schema, + e.default_version, + x.extversion AS installed_version, + e.comment +FROM + pg_available_extensions() e(name, default_version, comment) + LEFT JOIN pg_extension x ON e.name = x.extname + LEFT JOIN pg_namespace n ON x.extnamespace = n.oid; + +CREATE VIEW pgmeta_foreign_tables AS SELECT + c.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS name, + obj_description(c.oid) AS comment +FROM + pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE + c.relkind = 'f'; + +CREATE VIEW pgmeta_functions AS with functions as ( + select + *, + -- proargmodes is null when all arg modes are IN + coalesce( + p.proargmodes, + array_fill('i'::text, array[cardinality(coalesce(p.proallargtypes, p.proargtypes))]) + ) as arg_modes, + -- proargnames is null when all args are unnamed + coalesce( + p.proargnames, + array_fill(''::text, array[cardinality(coalesce(p.proallargtypes, p.proargtypes))]) + ) as arg_names, + -- proallargtypes is null when all arg modes are IN + coalesce(p.proallargtypes, p.proargtypes) as arg_types, + array_cat( + array_fill(false, array[pronargs - pronargdefaults]), + array_fill(true, array[pronargdefaults])) as arg_has_defaults + from + pg_proc as p + where + p.prokind = 'f' +) +select + f.oid::int8 as id, + n.nspname as schema, + f.proname as name, + l.lanname as language, + case + when l.lanname = 'internal' then '' + else f.prosrc + end as definition, + case + when l.lanname = 'internal' then f.prosrc + else pg_get_functiondef(f.oid) + end as complete_statement, + coalesce(f_args.args, '[]') as args, + pg_get_function_arguments(f.oid) as argument_types, + pg_get_function_identity_arguments(f.oid) as identity_argument_types, + f.prorettype::int8 as return_type_id, + pg_get_function_result(f.oid) as return_type, + nullif(rt.typrelid::int8, 0) as return_type_relation_id, + f.proretset as is_set_returning_function, + case + when f.provolatile = 'i' then 'IMMUTABLE' + when f.provolatile = 's' then 'STABLE' + when f.provolatile = 'v' then 'VOLATILE' + end as behavior, + f.prosecdef as security_definer, + f_config.config_params as config_params +from + functions f + left join pg_namespace n on f.pronamespace = n.oid + left join pg_language l on f.prolang = l.oid + left join pg_type rt on rt.oid = f.prorettype + left join ( + select + oid, + jsonb_object_agg(param, value) filter (where param is not null) as config_params + from + ( + select + oid, + (string_to_array(unnest(proconfig), '='))[1] as param, + (string_to_array(unnest(proconfig), '='))[2] as value + from + functions + ) as t + group by + oid + ) f_config on f_config.oid = f.oid + left join ( + select + oid, + jsonb_agg(jsonb_build_object( + 'mode', t2.mode, + 'name', name, + 'type_id', type_id, + 'has_default', has_default + )) as args + from + ( + select + oid, + unnest(arg_modes) as mode, + unnest(arg_names) as name, + unnest(arg_types)::int8 as type_id, + unnest(arg_has_defaults) as has_default + from + functions + ) as t1, + lateral ( + select + case + when t1.mode = 'i' then 'in' + when t1.mode = 'o' then 'out' + when t1.mode = 'b' then 'inout' + when t1.mode = 'v' then 'variadic' + else 'table' + end as mode + ) as t2 + group by + t1.oid + ) f_args on f_args.oid = f.oid; + +CREATE VIEW pgmeta_materialized_views AS select + c.oid::int8 as id, + n.nspname as schema, + c.relname as name, + c.relispopulated as is_populated, + obj_description(c.oid) as comment +from + pg_class c + join pg_namespace n on n.oid = c.relnamespace +where + c.relkind = 'm'; + +CREATE VIEW pgmeta_policies AS SELECT + pol.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS table, + c.oid :: int8 AS table_id, + pol.polname AS name, + CASE + WHEN pol.polpermissive THEN 'PERMISSIVE' :: text + ELSE 'RESTRICTIVE' :: text + END AS action, + CASE + WHEN pol.polroles = '{0}' :: oid [] THEN array_to_json( + string_to_array('public' :: text, '' :: text) :: name [] + ) + ELSE array_to_json( + ARRAY( + SELECT + pg_roles.rolname + FROM + pg_roles + WHERE + pg_roles.oid = ANY (pol.polroles) + ORDER BY + pg_roles.rolname + ) + ) + END AS roles, + CASE + pol.polcmd + WHEN 'r' :: "char" THEN 'SELECT' :: text + WHEN 'a' :: "char" THEN 'INSERT' :: text + WHEN 'w' :: "char" THEN 'UPDATE' :: text + WHEN 'd' :: "char" THEN 'DELETE' :: text + WHEN '*' :: "char" THEN 'ALL' :: text + ELSE NULL :: text + END AS command, + pg_get_expr(pol.polqual, pol.polrelid) AS definition, + pg_get_expr(pol.polwithcheck, pol.polrelid) AS check +FROM + pg_policy pol + JOIN pg_class c ON c.oid = pol.polrelid + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace; + +CREATE VIEW pgmeta_primary_keys AS SELECT + n.nspname AS schema, + c.relname AS table_name, + a.attname AS name, + c.oid :: int8 AS table_id +FROM + pg_index i, + pg_class c, + pg_attribute a, + pg_namespace n +WHERE + i.indrelid = c.oid + AND c.relnamespace = n.oid + AND a.attrelid = c.oid + AND a.attnum = ANY (i.indkey) + AND i.indisprimary; + +CREATE VIEW pgmeta_publications AS SELECT + p.oid :: int8 AS id, + p.pubname AS name, + p.pubowner::regrole::text AS owner, + p.pubinsert AS publish_insert, + p.pubupdate AS publish_update, + p.pubdelete AS publish_delete, + p.pubtruncate AS publish_truncate, + CASE + WHEN p.puballtables THEN NULL + ELSE pr.tables + END AS tables +FROM + pg_catalog.pg_publication AS p + LEFT JOIN LATERAL ( + SELECT + COALESCE( + array_agg( + json_build_object( + 'id', + c.oid :: int8, + 'name', + c.relname, + 'schema', + nc.nspname + ) + ), + '{}' + ) AS tables + FROM + pg_catalog.pg_publication_rel AS pr + JOIN pg_class AS c ON pr.prrelid = c.oid + join pg_namespace as nc on c.relnamespace = nc.oid + WHERE + pr.prpubid = p.oid + ) AS pr ON 1 = 1; + +CREATE VIEW pgmeta_relationships AS SELECT + c.oid :: int8 AS id, + c.conname AS constraint_name, + nsa.nspname AS source_schema, + csa.relname AS source_table_name, + sa.attname AS source_column_name, + nta.nspname AS target_table_schema, + cta.relname AS target_table_name, + ta.attname AS target_column_name +FROM + pg_constraint c + JOIN ( + pg_attribute sa + JOIN pg_class csa ON sa.attrelid = csa.oid + JOIN pg_namespace nsa ON csa.relnamespace = nsa.oid + ) ON sa.attrelid = c.conrelid + AND sa.attnum = ANY (c.conkey) + JOIN ( + pg_attribute ta + JOIN pg_class cta ON ta.attrelid = cta.oid + JOIN pg_namespace nta ON cta.relnamespace = nta.oid + ) ON ta.attrelid = c.confrelid + AND ta.attnum = ANY (c.confkey) +WHERE + c.contype = 'f'; + +CREATE VIEW pgmeta_roles AS -- TODO: Consider using pg_authid vs. pg_roles for unencrypted password field +SELECT + oid :: int8 AS id, + rolname AS name, + rolsuper AS is_superuser, + rolcreatedb AS can_create_db, + rolcreaterole AS can_create_role, + rolinherit AS inherit_role, + rolcanlogin AS can_login, + rolreplication AS is_replication_role, + rolbypassrls AS can_bypass_rls, + ( + SELECT + COUNT(*) + FROM + pg_stat_activity + WHERE + pg_roles.rolname = pg_stat_activity.usename + ) AS active_connections, + CASE WHEN rolconnlimit = -1 THEN current_setting('max_connections') :: int8 + ELSE rolconnlimit + END AS connection_limit, + rolpassword AS password, + rolvaliduntil AS valid_until, + rolconfig AS config +FROM + pg_roles; + +CREATE VIEW pgmeta_schemas AS select + n.oid::int8 as id, + n.nspname as name, + u.rolname as owner +from + pg_namespace n, + pg_roles u +where + n.nspowner = u.oid + and ( + pg_has_role(n.nspowner, 'USAGE') + or has_schema_privilege(n.oid, 'CREATE, USAGE') + ) + and not pg_catalog.starts_with(n.nspname, 'pg_temp_') + and not pg_catalog.starts_with(n.nspname, 'pg_toast_temp_'); + +CREATE VIEW pgmeta_tables AS SELECT + c.oid :: int8 AS id, + nc.nspname AS schema, + c.relname AS name, + c.relrowsecurity AS rls_enabled, + c.relforcerowsecurity AS rls_forced, + CASE + WHEN c.relreplident = 'd' THEN 'DEFAULT' + WHEN c.relreplident = 'i' THEN 'INDEX' + WHEN c.relreplident = 'f' THEN 'FULL' + ELSE 'NOTHING' + END AS replica_identity, + pg_total_relation_size(format('%I.%I', nc.nspname, c.relname)) :: int8 AS bytes, + pg_size_pretty( + pg_total_relation_size(format('%I.%I', nc.nspname, c.relname)) + ) AS size, + pg_stat_get_live_tuples(c.oid) AS live_rows_estimate, + pg_stat_get_dead_tuples(c.oid) AS dead_rows_estimate, + obj_description(c.oid) AS comment +FROM + pg_namespace nc + JOIN pg_class c ON nc.oid = c.relnamespace +WHERE + c.relkind IN ('r', 'p') + AND NOT pg_is_other_temp_schema(nc.oid) + AND ( + pg_has_role(c.relowner, 'USAGE') + OR has_table_privilege( + c.oid, + 'SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER' + ) + OR has_any_column_privilege(c.oid, 'SELECT, INSERT, UPDATE, REFERENCES') + ); + + +CREATE VIEW pgmeta_triggers AS SELECT + pg_t.oid AS id, + pg_t.tgrelid AS table_id, + CASE + WHEN pg_t.tgenabled = 'D' THEN 'DISABLED' + WHEN pg_t.tgenabled = 'O' THEN 'ORIGIN' + WHEN pg_t.tgenabled = 'R' THEN 'REPLICA' + WHEN pg_t.tgenabled = 'A' THEN 'ALWAYS' + END AS enabled_mode, + ( + STRING_TO_ARRAY( + ENCODE(pg_t.tgargs, 'escape'), '\000' + ) + )[:pg_t.tgnargs] AS function_args, + is_t.trigger_name AS name, + is_t.event_object_table AS table, + is_t.event_object_schema AS schema, + is_t.action_condition AS condition, + is_t.action_orientation AS orientation, + is_t.action_timing AS activation, + ARRAY_AGG(is_t.event_manipulation)::text[] AS events, + pg_p.proname AS function_name, + pg_n.nspname AS function_schema +FROM + pg_trigger AS pg_t +JOIN + pg_class AS pg_c +ON pg_t.tgrelid = pg_c.oid +JOIN information_schema.triggers AS is_t +ON is_t.trigger_name = pg_t.tgname +AND pg_c.relname = is_t.event_object_table +JOIN pg_proc AS pg_p +ON pg_t.tgfoid = pg_p.oid +JOIN pg_namespace AS pg_n +ON pg_p.pronamespace = pg_n.oid +GROUP BY + pg_t.oid, + pg_t.tgrelid, + pg_t.tgenabled, + pg_t.tgargs, + pg_t.tgnargs, + is_t.trigger_name, + is_t.event_object_table, + is_t.event_object_schema, + is_t.action_condition, + is_t.action_orientation, + is_t.action_timing, + pg_p.proname, + pg_n.nspname; + + +CREATE VIEW pgmeta_types AS select + t.oid::int8 as id, + t.typname as name, + n.nspname as schema, + format_type (t.oid, null) as format, + coalesce(t_enums.enums, '[]') as enums, + coalesce(t_attributes.attributes, '[]') as attributes, + obj_description (t.oid, 'pg_type') as comment +from + pg_type t + left join pg_namespace n on n.oid = t.typnamespace + left join ( + select + enumtypid, + jsonb_agg(enumlabel order by enumsortorder) as enums + from + pg_enum + group by + enumtypid + ) as t_enums on t_enums.enumtypid = t.oid + left join ( + select + oid, + jsonb_agg( + jsonb_build_object('name', a.attname, 'type_id', a.atttypid::int8) + order by a.attnum asc + ) as attributes + from + pg_class c + join pg_attribute a on a.attrelid = c.oid + where + c.relkind = 'c' and not a.attisdropped + group by + c.oid + ) as t_attributes on t_attributes.oid = t.typrelid +where + ( + t.typrelid = 0 + or ( + select + c.relkind = 'c' + from + pg_class c + where + c.oid = t.typrelid + ) + ); + +CREATE VIEW pgmeta_version AS SELECT + version(), + current_setting('server_version_num') :: int8 AS version_number, + ( + SELECT + COUNT(*) AS active_connections + FROM + pg_stat_activity + ) AS active_connections, + current_setting('max_connections') :: int8 AS max_connections; + +CREATE VIEW pgmeta_views AS SELECT + c.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS name, + -- See definition of information_schema.views + (pg_relation_is_updatable(c.oid, false) & 20) = 20 AS is_updatable, + obj_description(c.oid) AS comment +FROM + pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE + c.relkind = 'v'; +$adminpack$, + +$description_md$ +# michelp@adminpack + +A Trusted Language Extension containing a variety of useful databse admin queries. + +Postgres database admins are often faced with a variety of issues that +require them introspecting the database state. This extension +contains a mixed-bag of views that reflect useful information for +admins. + +## Installation + +Using dbdev: + +``` +postgres=> select dbdev.install('michelp@adminpack'); + install +--------- + t +(1 row) + +postgres=> create schema adminpack; +CREATE SCHEMA +postgres=> create extension "michelp@adminpack" with schema adminpack; +CREATE EXTENSION +``` + +This will the extension into the `adminpack` schema, substitute with +some other schema if you want it to go somewhere else. + +## Index Bloat + +Index updates and querying can end up getting slower and slower over +time due to what's called "index bloat". This is where frequent +updates and deletes can cause space in index data structures to go +unused. Understanding when this is happening is not obvious, so there +is a rather complex query you can run to detect it. + +`index_bloat` + +| Column | Type | +|------------------|------------------| +| current_database | name | +| schemaname | name | +| tblname | name | +| idxname | name | +| real_size | numeric | +| extra_size | numeric | +| extra_pct | double precision | +| fillfactor | integer | +| bloat_size | double precision | +| bloat_pct | double precision | +| is_na | boolean | + + +## Table Bloat + +Like index bloat, tables with high update and delete rates can also +end up containing lots of allocated but unused space. This query +shows which tables have the most bloat. + +`table_bloat` + +| Column | Type | +|------------------|------------------| +| current_database | name | +| schemaname | name | +| tblname | name | +| real_size | numeric | +| extra_size | double precision | +| extra_pct | double precision | +| fillfactor | integer | +| bloat_size | double precision | +| bloat_pct | double precision | +| is_na | boolean | + + +## Blocking PID Tree + +Postgres queries can block each other, and this blocking relationship +can form a tree, where A blocks B, which blocks C, and so on. This +query formats blocking queries into a tree structure so it's easy to +see what query is causing the blockage. + +`blocking_pid_tree` + +| Column | Type | +|-----------|------| +| PID | text | +| Lock Info | text | +| State | text | + + +## Duplicate Indexes + +If a table contains duplicate indexes, then unecessary work is done +updating and storing them, this query will show up to 4 duplicate +indexes per table if they exist. + +`duplicate_indexes` + +| Column | Type | +|--------|----------| +| size | text | +| idx1 | regclass | +| idx2 | regclass | +| idx3 | regclass | +| idx4 | regclass | + + +## Table Sizes + +This view shows tables and their sizes. + +`table_sizes` + +| Column | Type | +|------------------|------------------| +| table_schema | name | +| table_name | name | +| row_estimate | real | +| total | text | +| index | text | +| toast | text | +| table | text | +| total_size_share | double precision | + + +## Schema Sizes + +This view shows schemas and their sizes, which is the sum of the sizes +of all the tables and indexes in the schema. + +`schema_sizes` + +| Column | Type | +|-------------|---------| +| schemaname | name | +| table | text | +| index | text | +| table_index | text | +| sum | numeric | + + +## Index Usage + +This view shows index size and usage. An unused index still needs to +be updated and that takes time an storage, so it's a good candidate to +drop. + +`index_usage` + +| Column | Type | +|-----------------|--------| +| schemaname | name | +| tablename | name | +| num_rows | bigint | +| table_size | text | +| index_name | name | +| index_size | text | +| unique | text | +| number_of_scans | bigint | +| tuples_read | bigint | +| tuples_fetched | bigint | + + +## Last Vacuum Analyze + +This views shows the last time a table was vacuumed an analyzed. + +`last_vacuum_analyze` + +| Column | Type | +|---------------------|--------------------------| +| relname | name | +| last_vacuum | timestamp with time zone | +| n_mod_since_analyze | timestamp with time zone | +| last_analyze | timestamp with time zone | +| last_autoanalyze | timestamp with time zone | +| analyze_count | bigint | +| autoanalyze_count | bigint | + +## Table Row Estimates + +This view shows estimates for the number of rows in a table. This is +just an estimate and depends on up to date table statistics. + +`table_row_estimates` + +| Column | Type | +|------------|--------| +| schemaname | name | +| relname | name | +| n_live_tup | bigint | + +## PGMeta Columns + +This view shows infromation for the columns of tables in the system. + +`pgmeta_columns` + +| Column | Type | +|---------------------|----------| +| table_id | bigint | +| schema | name | +| table | name | +| id | text | +| ordinal_position | smallint | +| name | name | +| default_value | text | +| data_type | text | +| format | name | +| is_identity | boolean | +| identity_generation | text | +| is_generated | boolean | +| is_nullable | boolean | +| is_updatable | boolean | +| is_unique | boolean | +| enums | json | +| comment | text | + +## PGMeta Config + +This views shows the configuration of the database. + +`pgmeta_config` + +| Column | Type | +|-----------------|---------| +| name | text | +| setting | text | +| category | text | +| group | text | +| subgroup | text | +| unit | text | +| short_desc | text | +| extra_desc | text | +| context | text | +| vartype | text | +| source | text | +| min_val | text | +| max_val | text | +| enumvals | text[] | +| boot_val | text | +| reset_val | text | +| sourcefile | text | +| sourceline | integer | +| pending_restart | boolean | + +## PGMeta Extensions + +This view shows installed extensions in the database. + +`pgmeta_extensions` + +| Column | Type | +|-------------------|------| +| name | name | +| schema | name | +| default_version | text | +| installed_version | text | +| comment | text | + +## PGMeta Foreign Tables + +This view shows foreign tables in the database. + +`pgmeta_foreign_tables` + +| Column | Type | +|---------|--------| +| id | bigint | +| schema | name | +| name | name | +| comment | text | + +## PGMeta Functions + +This view shows functions in the database. + +`pgmeta_functions` + +| Column | Type | +|---------------------------|---------| +| id | bigint | +| schema | name | +| name | name | +| language | name | +| definition | text | +| complete_statement | text | +| args | jsonb | +| argument_types | text | +| identity_argument_types | text | +| return_type_id | bigint | +| return_type | text | +| return_type_relation_id | bigint | +| is_set_returning_function | boolean | +| behavior | text | +| security_definer | boolean | +| config_params | jsonb | + +## PGMeta Materialized Views + +This view shows materialized views in the database. + +`pgmeta_materialized_views` + +| Column | Type | +|--------------|---------| +| id | bigint | +| schema | name | +| name | name | +| is_populated | boolean | +| comment | text | + +## PGMeta Policies + +This view shows Row Level Security Policies in the database. + +`pgmeta_policies` + +| Column | Type | +|------------|--------| +| id | bigint | +| schema | name | +| table | name | +| table_id | bigint | +| name | name | +| action | text | +| roles | json | +| command | text | +| definition | text | +| check | text | + +## PGMeta Primary Keys + +This view shows primary keys in the database. + +`pgmeta_primary_keys` + +| Column | Type | +|------------|--------| +| schema | name | +| table_name | name | +| name | name | +| table_id | bigint | + +## PGMeta Publications + +This view shows logical replication publishers in the database. + +`pgmeta_publications` + +| Column | Type | +|------------------|---------| +| id | bigint | +| name | name | +| owner | text | +| publish_insert | boolean | +| publish_update | boolean | +| publish_delete | boolean | +| publish_truncate | boolean | +| tables | json[] | + +## PGMeta Relationships + +This view shows foreign key relationships in the database. + +`pgmeta_relationships` + +| Column | Type | +|---------------------|--------| +| id | bigint | +| constraint_name | name | +| source_schema | name | +| source_table_name | name | +| source_column_name | name | +| target_table_schema | name | +| target_table_name | name | +| target_column_name | name | + +## PGMeta Roles + +This view shows roles in the database system. Note that roles are +global objects and apply to all databases. + +`pgmeta_roles` + +| Column | Type | +|---------------------|--------------------------| +| id | bigint | +| name | name | +| is_superuser | boolean | +| can_create_db | boolean | +| can_create_role | boolean | +| inherit_role | boolean | +| can_login | boolean | +| is_replication_role | boolean | +| can_bypass_rls | boolean | +| active_connections | bigint | +| connection_limit | bigint | +| password | text | +| valid_until | timestamp with time zone | +| config | text[] | + +## PGMeta Schemas + +This view shows all schemas in the database. + +`pgmeta_schemas` + +| Column | Type | +|--------|--------| +| id | bigint | +| name | name | +| owner | name | + +## PGMeta Tables + +This view shows all tables in the database. + +`pgmeta_tables` + +| Column | Type | +|--------------------|---------| +| id | bigint | +| schema | name | +| name | name | +| rls_enabled | boolean | +| rls_forced | boolean | +| replica_identity | text | +| bytes | bigint | +| size | text | +| live_rows_estimate | bigint | +| dead_rows_estimate | bigint | +| comment | text | + +## PGMeta Triggers + +This view shows all triggers in the database. + +`pgmeta_triggers` + +| Column | Type | +|-----------------|-----------------------------------| +| id | oid | +| table_id | oid | +| enabled_mode | text | +| function_args | text[] | +| name | information_schema.sql_identifier | +| table | information_schema.sql_identifier | +| schema | information_schema.sql_identifier | +| condition | information_schema.character_data | +| orientation | information_schema.character_data | +| activation | information_schema.character_data | +| events | text[] | +| function_name | name | +| function_schema | name | + +## PGMeta Types + +This view shows all types in the database. + +`pgmeta_types` + +| Column | Type | +|------------|--------| +| id | bigint | +| name | name | +| schema | name | +| format | text | +| enums | jsonb | +| attributes | jsonb | +| comment | text | + +## PGMeta Version + +This view shows the current database version. + +`pgmeta_version` + +| Column | Type | +|--------------------|--------| +| version | text | +| version_number | bigint | +| active_connections | bigint | +| max_connections | bigint | + +## PGMeta Views + +This view shows all views in the database. + +`pgmeta_views` + +| Column | Type | +|--------------|---------| +| id | bigint | +| schema | name | +| name | name | +| is_updatable | boolean | +| comment | text | +$description_md$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20231207113329_olirice@index_advisor-0.2.1.sql b/crates/pgls_pretty_print/tests/data/multi/20231207113329_olirice@index_advisor-0.2.1.sql new file mode 100644 index 000000000..4efa51a4f --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20231207113329_olirice@index_advisor-0.2.1.sql @@ -0,0 +1,343 @@ +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_alias = 'olirice@index_advisor'), +(0,2,1), +$pkg$ + +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + hypopg_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = 'hypopg' + and installed_version is not null + ); + begin + + if not hypopg_exists then + raise + exception '"olirice@index_advisor" requires "hypopg"' + using hint = 'Run "create extension hypopg" and try again'; + end if; + end +$$; + +create or replace function index_advisor( + query text +) + returns table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[], + errors text[] + ) + volatile + language plpgsql + as $$ +declare + n_args int; + prepared_statement_name text = 'index_advisor_working_statement'; + hypopg_schema_name text = (select extnamespace::regnamespace::text from pg_extension where extname = 'hypopg'); + explain_plan_statement text; + error_message text; + rec record; + plan_initial jsonb; + plan_final jsonb; + statements text[] = '{}'; +begin + + -- Remove comment lines (its common that they contain semicolons) + query := trim( + regexp_replace( + regexp_replace( + regexp_replace(query,'\/\*.+\*\/', '', 'g'), + '--[^\r\n]*', ' ', 'g'), + '\s+', ' ', 'g') + ); + + -- Remove trailing semicolon + query := regexp_replace(query, ';\s*$', ''); + + begin + -- Disallow multiple statements + if query ilike '%;%' then + raise exception 'Query must not contain a semicolon'; + end if; + + -- Hack to support PostgREST because the prepared statement for args incorrectly defaults to text + query := replace(query, 'WITH pgrst_payload AS (SELECT $1 AS json_data)', 'WITH pgrst_payload AS (SELECT $1::json AS json_data)'); + + -- Create a prepared statement for the given query + deallocate all; + execute format('prepare %I as %s', prepared_statement_name, query); + + -- Detect how many arguments are present in the prepared statement + n_args = ( + select + coalesce(array_length(parameter_types, 1), 0) + from + pg_prepared_statements + where + name = prepared_statement_name + limit + 1 + ); + + -- Create a SQL statement that can be executed to collect the explain plan + explain_plan_statement = format( + 'set local plan_cache_mode = force_generic_plan; explain (format json) execute %I%s', + --'explain (format json) execute %I%s', + prepared_statement_name, + case + when n_args = 0 then '' + else format( + '(%s)', array_to_string(array_fill('null'::text, array[n_args]), ',') + ) + end + ); + + -- Store the query plan before any new indexes + execute explain_plan_statement into plan_initial; + + -- Create possible indexes + for rec in ( + with extension_regclass as ( + select + distinct objid as oid + from + pg_catalog.pg_depend + where + deptype = 'e' + ) + select + pc.relnamespace::regnamespace::text as schema_name, + pc.relname as table_name, + pa.attname as column_name, + format( + 'select %I.hypopg_create_index($i$create index on %I.%I(%I)$i$)', + hypopg_schema_name, + pc.relnamespace::regnamespace::text, + pc.relname, + pa.attname + ) hypopg_statement + from + pg_catalog.pg_class pc + join pg_catalog.pg_attribute pa + on pc.oid = pa.attrelid + left join extension_regclass er + on pc.oid = er.oid + left join pg_catalog.pg_index pi + on pc.oid = pi.indrelid + and (select array_agg(x) from unnest(pi.indkey) v(x)) = array[pa.attnum] + and pi.indexprs is null -- ignore expression indexes + and pi.indpred is null -- ignore partial indexes + where + pc.relnamespace::regnamespace::text not in ( -- ignore schema list + 'pg_catalog', 'pg_toast', 'information_schema' + ) + and er.oid is null -- ignore entities owned by extensions + and pc.relkind in ('r', 'm') -- regular tables, and materialized views + and pc.relpersistence = 'p' -- permanent tables (not unlogged or temporary) + and pa.attnum > 0 + and not pa.attisdropped + and pi.indrelid is null + and pa.atttypid in (20,16,1082,1184,1114,701,23,21,700,1083,2950,1700,25,18,1042,1043) + ) + loop + -- Create the hypothetical index + execute rec.hypopg_statement; + end loop; + + -- Create a prepared statement for the given query + -- The original prepared statement MUST be dropped because its plan is cached + execute format('deallocate %I', prepared_statement_name); + execute format('prepare %I as %s', prepared_statement_name, query); + + -- Store the query plan after new indexes + execute explain_plan_statement into plan_final; + + --raise notice '%', plan_final; + + -- Idenfity referenced indexes in new plan + execute format( + 'select + coalesce(array_agg(hypopg_get_indexdef(indexrelid) order by indrelid, indkey::text), $i${}$i$::text[]) + from + %I.hypopg() + where + %s ilike ($i$%%$i$ || indexname || $i$%%$i$) + ', + hypopg_schema_name, + quote_literal(plan_final)::text + ) into statements; + + -- Reset all hypothetical indexes + perform hypopg_reset(); + + -- Reset prepared statements + deallocate all; + + return query values ( + (plan_initial -> 0 -> 'Plan' -> 'Startup Cost'), + (plan_final -> 0 -> 'Plan' -> 'Startup Cost'), + (plan_initial -> 0 -> 'Plan' -> 'Total Cost'), + (plan_final -> 0 -> 'Plan' -> 'Total Cost'), + statements::text[], + array[]::text[] + ); + return; + + exception when others then + get stacked diagnostics error_message = MESSAGE_TEXT; + + return query values ( + null::jsonb, + null::jsonb, + null::jsonb, + null::jsonb, + array[]::text[], + array[error_message]::text[] + ); + return; + end; + +end; +$$; + +$pkg$, + +$description_md$ + +# index_advisor + +`index_advisor` is an extension for recommending indexes to improve query performance. + +## Installation + +Note: + +`hypopg` is a dependency of index_advisor. +Dependency resolution is currently under development. +In the future it will not be necessary to manually create dependencies. + + +```sql +select dbdev.install('olirice@index_advisor'); +create extension if not exists hypopg; +create extension "olirice@index_advisor" version '0.2.0'; +``` + +Features: +- Supports generic parameters e.g. `$1`, `$2` +- Supports materialized views +- Identifies tables/columns obfuscaed by views + + +## API + +#### Description +For a given *query*, searches for a set of SQL DDL `create index` statements that improve the query's execution time; + +#### Signature +```sql +index_advisor(query text) +returns + table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[], + errors text[] + ) +``` + +## Usage + +For a minimal example, the `index_advisor` function can be given a single table query with a filter on an unindexed column. + +```sql +create extension if not exists index_advisor cascade; + +create table book( + id int primary key, + title text not null +); + +select + * +from + index_advisor('select book.id from book where title = $1'); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements | errors +---------------------+--------------------+-------------------+------------------+-----------------------------------------------------+-------- + 0.00 | 1.17 | 25.88 | 6.40 | {"CREATE INDEX ON public.book USING btree (title)"},| {} +(1 row) +``` + +More complex queries may generate additional suggested indexes + +```sql +create extension if not exists index_advisor cascade; + +create table author( + id serial primary key, + name text not null +); + +create table publisher( + id serial primary key, + name text not null, + corporate_address text +); + +create table book( + id serial primary key, + author_id int not null references author(id), + publisher_id int not null references publisher(id), + title text +); + +create table review( + id serial primary key, + book_id int references book(id), + body text not null +); + +select + * +from + index_advisor(' + select + book.id, + book.title, + publisher.name as publisher_name, + author.name as author_name, + review.body review_body + from + book + join publisher + on book.publisher_id = publisher.id + join author + on book.author_id = author.id + join review + on book.id = review.book_id + where + author.id = $1 + and publisher.id = $2 + '); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements | errors +---------------------+--------------------+-------------------+------------------+-----------------------------------------------------------+-------- + 27.26 | 12.77 | 68.48 | 42.37 | {"CREATE INDEX ON public.book USING btree (author_id)", | {} + "CREATE INDEX ON public.book USING btree (publisher_id)", + "CREATE INDEX ON public.review USING btree (book_id)"} +(3 rows) +``` +$description_md$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20231207113857_olirice@read_once-0.3.2.sql b/crates/pgls_pretty_print/tests/data/multi/20231207113857_olirice@read_once-0.3.2.sql new file mode 100644 index 000000000..c8d831a27 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20231207113857_olirice@read_once-0.3.2.sql @@ -0,0 +1,145 @@ +insert into app.package_versions(package_id, version_struct, sql, description_md) +values ( +(select id from app.packages where package_alias = 'olirice@read_once'), +(0,3,2), +$pkg$ + +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + pg_cron_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = 'pg_cron' + and installed_version is not null + ); + begin + + if not pg_cron_exists then + raise + exception '"olirice@read_once" requires "pg_cron"' + using hint = 'Run "create extension pg_cron" and try again'; + end if; + end +$$; + + +create schema read_once; + +create unlogged table read_once.messages( + id uuid primary key default gen_random_uuid(), + contents text not null default '', + created_at timestamp default now() +); + +revoke all on read_once.messages from public; +revoke usage on schema read_once from public; + +create or replace function send_message( + contents text +) + returns uuid + security definer + volatile + strict + language sql + as +$$ + insert into read_once.messages(contents) + values ($1) + returning id; +$$; + +create or replace function read_message(id uuid) + returns text + security definer + volatile + strict + language sql + as +$$ + delete from read_once.messages + where read_once.messages.id = $1 + returning contents; +$$ +$pkg$, + +$description_md$ + +# read_once + +A Supabase application for sending messages that can only be read once + +Features: +- messages can only be read one time +- messages are not logged in PostgreSQL write-ahead-log (WAL) + +## Installation + +`pg_cron` is a dependency of `read_once`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + +To expose the `send_message` and `read_message` functions over HTTP, install the extension in a schema that is on the search_path. + + +For example: +```sql +select dbdev.install('olirice@read_once'); +create extension if not exists pg_cron; +create extension "olirice@read_once" + schema public + version '0.3.1'; +``` + + +## HTTP API + +### Create a Message + +```sh +curl -X POST https://.supabase.co/rest/v1/rpc/send_message \ + -H 'apiKey: ' \ + -H 'Content-Type: application/json' + --data-raw '{"contents": "hello, dbdev!"} + +# Returns: "2989156b-2356-4543-9d1b-19dfb8ec3268" +``` + +### Read a Message + +```sh +curl -X https://.supabase.co/rest/v1/rpc/read_message + -H 'apiKey: ' \ + -H 'Content-Type: application/json' \ + --data-raw '{"id": "2989156b-2356-4543-9d1b-19dfb8ec3268"} + +# Returns: "hello, dbdev!" +``` + + +## SQL API + +### Create a Message + +```sql +-- Creates a new messages and returns its unique id +create or replace function send_message( + contents text +) + returns uuid +``` + +### Read a Message + +```sql +-- Read a message by its id +create or replace function read_message( + id uuid +) + returns text +``` +$description_md$ +); diff --git a/crates/pgls_pretty_print/tests/data/multi/20240108072747_update_provider_id.sql b/crates/pgls_pretty_print/tests/data/multi/20240108072747_update_provider_id.sql new file mode 100644 index 000000000..cf923d2f5 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20240108072747_update_provider_id.sql @@ -0,0 +1,8 @@ +-- For email provider the provider_id should be the lowercase email +-- which is availble in the email column +-- This migration was necessecitated by a recent change in identities +-- table schema by gotrue: +-- https://github.com/supabase/gotrue/blob/master/migrations/20231117164230_add_id_pkey_identities.up.sql +update auth.identities +set provider_id = email +where provider = 'email'; diff --git a/crates/pgls_pretty_print/tests/data/multi/20240605122023_fix_view_permissions.sql b/crates/pgls_pretty_print/tests/data/multi/20240605122023_fix_view_permissions.sql new file mode 100644 index 000000000..50d0d6aa3 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20240605122023_fix_view_permissions.sql @@ -0,0 +1,26 @@ +-- set view security_invoker=true to fix linter errors +alter view public.packages set (security_invoker=true); +alter view public.package_versions set (security_invoker=true); +alter view public.package_upgrades set (security_invoker=true); + +-- create policies to allow anon role to read from the views +create policy packages_select_policy_anon + on app.packages + as permissive + for select + to anon + using (true); + +create policy package_versions_select_policy_anon + on app.package_versions + as permissive + for select + to anon + using (true); + +create policy package_upgrades_select_policy_anon + on app.package_upgrades + as permissive + for select + to anon + using (true); diff --git a/crates/pgls_pretty_print/tests/data/multi/20240705083738_remove_contact_email.sql b/crates/pgls_pretty_print/tests/data/multi/20240705083738_remove_contact_email.sql new file mode 100644 index 000000000..f216f8fce --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20240705083738_remove_contact_email.sql @@ -0,0 +1,33 @@ +drop view if exists public.accounts; + +create view + public.accounts +with + (security_invoker = true) as +select + acc.id, + acc.handle, + obj.name as avatar_path, + acc.display_name, + acc.bio, + acc.created_at +from + app.accounts acc + left join storage.objects obj on acc.avatar_id = obj.id; + +drop view if exists public.organizations; + +create view + public.organizations +with + (security_invoker = true) as +select + org.id, + org.handle, + obj.name as avatar_path, + org.display_name, + org.bio, + org.created_at +from + app.organizations org + left join storage.objects obj on org.avatar_id = obj.id; diff --git a/crates/pgls_pretty_print/tests/data/multi/20250106073735_jwt_secret_from_vault.sql b/crates/pgls_pretty_print/tests/data/multi/20250106073735_jwt_secret_from_vault.sql new file mode 100644 index 000000000..361323c2c --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20250106073735_jwt_secret_from_vault.sql @@ -0,0 +1,66 @@ +-- app.settings.jwt_secret has been removed, see https://github.com/orgs/supabase/discussions/30606 +-- now we fetch the secret from vault. The redeem_access_token function is the same as before +-- (see file 20230906110845_access_token.sql) except the part where we fetch the jwt_secret. +create or replace function public.redeem_access_token( + access_token text +) + returns text + language plpgsql + security definer + strict +as $$ +declare + token_id uuid; + token bytea; + tokens_row app.user_id_and_token_hash; + token_valid boolean; + now timestamp; + one_hour_from_now timestamp; + issued_at int; + expiry_at int; + jwt_secret text; +begin + -- validate access token + if length(access_token) != 64 then + raise exception 'Invalid token'; + end if; + + if substring(access_token from 1 for 4) != 'dbd_' then + raise exception 'Invalid token'; + end if; + + token_id := substring(access_token from 5 for 32)::uuid; + token := app.base64url_decode(substring(access_token from 37)); + + select t.user_id, t.token_hash + into tokens_row + from app.access_tokens t + where t.id = token_id; + + -- TODO: do a constant time comparison + if tokens_row.token_hash != sha256(token) then + raise exception 'Invalid token'; + end if; + + -- Generate JWT token + now := current_timestamp; + one_hour_from_now := now + interval '1 hour'; + issued_at := date_part('epoch', now); + expiry_at := date_part('epoch', one_hour_from_now); + + -- read the jwt secret from vault + select decrypted_secret + into jwt_secret + from vault.decrypted_secrets + where name = 'app.jwt_secret'; + + return sign(json_build_object( + 'aud', 'authenticated', + 'role', 'authenticated', + 'iss', 'database.dev', + 'sub', tokens_row.user_id, + 'iat', issued_at, + 'exp', expiry_at + ), jwt_secret); +end; +$$; diff --git a/crates/pgls_pretty_print/tests/data/multi/20250217100252_restrict_accounts_and_orgs.sql b/crates/pgls_pretty_print/tests/data/multi/20250217100252_restrict_accounts_and_orgs.sql new file mode 100644 index 000000000..4f6e18a1a --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20250217100252_restrict_accounts_and_orgs.sql @@ -0,0 +1,49 @@ +-- Only allow authenticated users to view their own accounts. +alter policy accounts_select_policy + on app.accounts + to authenticated + using (id = auth.uid()); + +-- Only allow organization maintainers to view their own organizations. +alter policy organizations_select_policy + on app.organizations + to authenticated + using (app.is_organization_maintainer(auth.uid(), id)); + +-- Allow authenticated users to get an account by handle. +create or replace function public.get_account( + handle text +) + returns setof public.accounts + language sql + security definer + strict +as $$ + select id, handle, avatar_path, display_name, bio, created_at + from public.accounts a + where a.handle = get_account.handle + and auth.uid() is not null; +$$; + +-- Allow authenticated users to get an organization by handle. +create or replace function public.get_organization( + handle text +) + returns setof public.organizations + language sql + security definer + strict +as $$ + select id, handle, avatar_path, display_name, bio, created_at + from public.organizations o + where o.handle = get_organization.handle + and auth.uid() is not null; +$$; + +-- Allow service role to read all accounts and organizations. +grant select on app.accounts to service_role; +grant select on app.organizations to service_role; + +-- Allow service role to read all packages. +grant select on app.packages to service_role; +grant select on app.package_versions to service_role; diff --git a/crates/pgls_pretty_print/tests/data/multi/20250804111152_remove_dbdev_from_popular_packages.sql b/crates/pgls_pretty_print/tests/data/multi/20250804111152_remove_dbdev_from_popular_packages.sql new file mode 100644 index 000000000..1a8333e16 --- /dev/null +++ b/crates/pgls_pretty_print/tests/data/multi/20250804111152_remove_dbdev_from_popular_packages.sql @@ -0,0 +1,12 @@ +create or replace function public.popular_packages() +returns setof public.packages +language sql stable +as $$ + select * from public.packages p + where p.package_name != 'supabase-dbdev' + order by ( + select (dm.downloads_30_day * 5) + (dm.downloads_90_days * 2) + dm.downloads_180_days + from public.download_metrics dm + where dm.package_id = p.id + ) desc nulls last, p.created_at desc; +$$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141357_extensions_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141357_extensions_100.snap new file mode 100644 index 000000000..7b1e65ddb --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141357_extensions_100.snap @@ -0,0 +1,11 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117141357_extensions.sql +--- +create extension if not exists pg_stat_statements with schema extensions; + +create extension if not exists pg_trgm with schema extensions; + +create extension if not exists citext with schema extensions; + +create extension if not exists pg_cron; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141357_extensions_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141357_extensions_80.snap new file mode 100644 index 000000000..7b1e65ddb --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141357_extensions_80.snap @@ -0,0 +1,11 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117141357_extensions.sql +--- +create extension if not exists pg_stat_statements with schema extensions; + +create extension if not exists pg_trgm with schema extensions; + +create extension if not exists citext with schema extensions; + +create extension if not exists pg_cron; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141359_app_schema_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141359_app_schema_100.snap new file mode 100644 index 000000000..929b8423a --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141359_app_schema_100.snap @@ -0,0 +1,9 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117141359_app_schema.sql +--- +create schema "app"; + +grant USAGE on schema app to authenticated, anon; + +alter default privileges in schema app grant SELECT on tables to authenticated, anon; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141359_app_schema_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141359_app_schema_80.snap new file mode 100644 index 000000000..f3a51a785 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141359_app_schema_80.snap @@ -0,0 +1,14 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117141359_app_schema.sql +--- +create schema "app"; + +grant USAGE on schema app to authenticated, anon; + +alter default privileges +in schema app +grant SELECT +on tables +to authenticated, +anon; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141507_semver_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141507_semver_100.snap new file mode 100644 index 000000000..3a7107c0e --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141507_semver_100.snap @@ -0,0 +1,60 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117141507_semver.sql +--- +create type app.semver_struct as (major smallint, minor smallint, patch smallint); + +create or replace function app.is_valid(app.semver_struct) +returns boolean +language sql +immutable +as $function$ +select ( + ($1).major is not null + and ($1).minor is not null + and ($1).patch is not null + ) +$function$; + +create domain app.semver as app.semver_struct check (app.is_valid(value)); + +create function app.semver_exception(version text) +returns app.semver_struct +language plpgsql +immutable +as $function$ +begin + raise exception using errcode='22000', message=format('Invalid semver %L', version); +end; +$function$; + +create function app.text_to_semver(text) +returns app.semver_struct +language sql +immutable +strict +as $function$ +with s(version) as ( + select ( + split_part($1, '.', 1), + split_part($1, '.', 2), + split_part(split_part(split_part($1, '.', 3), '-', 1), '+', 1) + )::app.semver_struct + ) + select + case app.is_valid(s.version) + when true then s.version + else app.semver_exception($1) + end + from + s +$function$; + +create or replace function app.semver_to_text(app.semver) +returns text +language sql +immutable +as $function$ +select + format('%s.%s.%s', $1.major, $1.minor, $1.patch) +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141507_semver_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141507_semver_80.snap new file mode 100644 index 000000000..559008859 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141507_semver_80.snap @@ -0,0 +1,64 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117141507_semver.sql +--- +create type app.semver_struct as ( + major smallint, + minor smallint, + patch smallint +); + +create or replace function app.is_valid(app.semver_struct) +returns boolean +language sql +immutable +as $function$ +select ( + ($1).major is not null + and ($1).minor is not null + and ($1).patch is not null + ) +$function$; + +create domain app.semver as app.semver_struct check (app.is_valid(value)); + +create function app.semver_exception(version text) +returns app.semver_struct +language plpgsql +immutable +as $function$ +begin + raise exception using errcode='22000', message=format('Invalid semver %L', version); +end; +$function$; + +create function app.text_to_semver(text) +returns app.semver_struct +language sql +immutable +strict +as $function$ +with s(version) as ( + select ( + split_part($1, '.', 1), + split_part($1, '.', 2), + split_part(split_part(split_part($1, '.', 3), '-', 1), '+', 1) + )::app.semver_struct + ) + select + case app.is_valid(s.version) + when true then s.version + else app.semver_exception($1) + end + from + s +$function$; + +create or replace function app.semver_to_text(app.semver) +returns text +language sql +immutable +as $function$ +select + format('%s.%s.%s', $1.major, $1.minor, $1.patch) +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141645_valid_name_type_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141645_valid_name_type_100.snap new file mode 100644 index 000000000..e3dc2bcc6 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141645_valid_name_type_100.snap @@ -0,0 +1,16 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117141645_valid_name_type.sql +--- +create extension if not exists citext with schema extensions; + +create domain app.valid_name as extensions.citext check (value ~ '^[A-z][A-z0-9\_]{2,32}$'); + +create or replace function app.exception(message text) +returns text +language plpgsql +as $function$ +begin + raise exception using errcode='22000', message=message; + end; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141645_valid_name_type_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141645_valid_name_type_80.snap new file mode 100644 index 000000000..d683e47ed --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141645_valid_name_type_80.snap @@ -0,0 +1,17 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117141645_valid_name_type.sql +--- +create extension if not exists citext with schema extensions; + +create domain app.valid_name as extensions.citext + check (value ~ '^[A-z][A-z0-9\_]{2,32}$'); + +create or replace function app.exception(message text) +returns text +language plpgsql +as $function$ +begin + raise exception using errcode='22000', message=message; + end; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141942_email_address_type_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141942_email_address_type_100.snap new file mode 100644 index 000000000..f686f187f --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141942_email_address_type_100.snap @@ -0,0 +1,7 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117141942_email_address_type.sql +--- +create domain app.email_address as citext + check (value ~ + '^[a-zA-Z0-9.!#$%&''*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$'); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141942_email_address_type_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141942_email_address_type_80.snap new file mode 100644 index 000000000..f686f187f --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117141942_email_address_type_80.snap @@ -0,0 +1,7 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117141942_email_address_type.sql +--- +create domain app.email_address as citext + check (value ~ + '^[a-zA-Z0-9.!#$%&''*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$'); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142104_account_and_org_tables_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142104_account_and_org_tables_100.snap new file mode 100644 index 000000000..4d14b9ab8 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142104_account_and_org_tables_100.snap @@ -0,0 +1,162 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117142104_account_and_org_tables.sql +--- +insert into storage.buckets (id, name) values ('avatars', 'avatars'); + +create table app.handle_registry ( + handle app.valid_name primary key not null, + is_organization boolean not null, + created_at timestamp with time zone + not null + default NOW(), + unique (handle, is_organization) +); + +create table app.accounts ( + id uuid primary key references auth.users (id), + handle app.valid_name not null unique, + is_organization boolean + generated always as (false) stored, + avatar_id uuid references storage.objects (id), + display_name text + check (length(display_name) <= 128), + bio text check (length(bio) <= 512), + contact_email app.email_address, + created_at timestamp with time zone + not null + default NOW(), + constraint "fk_handle_registry" + foreign key + (handle, + is_organization) + references app.handle_registry (handle, + is_organization) +); + +create or replace function app.register_account() +returns trigger +language plpgsql +security definer +as $function$ +begin + insert into app.handle_registry (handle, is_organization) + values ( + new.raw_user_meta_data ->> 'handle', + false + ); + + insert into app.accounts (id, handle, display_name, bio, contact_email) + values ( + new.id, + new.raw_user_meta_data ->> 'handle', + new.raw_user_meta_data ->> 'display_name', + new.raw_user_meta_data ->> 'bio', + new.raw_user_meta_data ->> 'contact_email' + ); + return new; + end; +$function$; + +create or replace trigger on_auth_user_created +after insert +on auth.users +for each row +execute function app.register_account(); + +create table app.organizations ( + id uuid primary key default gen_random_uuid(), + handle app.valid_name not null unique, + is_organization boolean + generated always as (true) stored, + avatar_id uuid references storage.objects (id), + display_name text + check (length(display_name) <= 128), + bio text check (length(bio) <= 512), + contact_email app.email_address, + created_at timestamp with time zone + not null + default NOW(), + constraint "fk_handle_registry" + foreign key + (handle, + is_organization) + references app.handle_registry (handle, + is_organization) +); + +create type app.membership_role as enum ('maintainer'); + +create table app.members ( + id uuid primary key default uuid_generate_v4(), + organization_id uuid + not null + references app.organizations (id), + account_id uuid + not null + references app.accounts (id), + role app.membership_role not null, + created_at timestamp with time zone + not null + default NOW(), + unique (organization_id, account_id) +); + +create or replace function app.register_organization_creator_as_member() +returns trigger +language plpgsql +security definer +as $function$ +begin + insert into app.members(organization_id, account_id, role) + values (new.id, auth.uid(), 'maintainer'); + + return new; + end; +$function$; + +create or replace trigger on_app_organization_created +after insert +on app.organizations +for each row +execute function app.register_organization_creator_as_member(); + +create or replace function app.update_avatar_id() +returns trigger +language plpgsql +security definer +as $function$ +declare + v_handle app.valid_name; + v_affected_account app.accounts := null; + begin + select (string_to_array(new.name, '-'::text))[1]::app.valid_name into v_handle; + + update app.accounts + set avatar_id = new.id + where handle = v_handle + returning * into v_affected_account; + + if not v_affected_account is null then + update auth.users u + set + "raw_user_meta_data" = u.raw_user_meta_data || jsonb_build_object( + 'avatar_path', new.name + ) + where u.id = v_affected_account.id; + else + update app.organizations + set avatar_id = new.id + where handle = v_handle; + end if; + + return new; + end; +$function$; + +create or replace trigger on_storage_object_created +after insert +on storage.objects +for each row +when (new.bucket_id = 'avatars') +execute function app.update_avatar_id(); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142104_account_and_org_tables_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142104_account_and_org_tables_80.snap new file mode 100644 index 000000000..b409db604 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142104_account_and_org_tables_80.snap @@ -0,0 +1,172 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117142104_account_and_org_tables.sql +--- +insert into storage.buckets (id, name) values ('avatars', 'avatars'); + +create table app.handle_registry ( + handle app.valid_name + primary key + not null, + is_organization boolean not null, + created_at timestamp with time zone + not null + default NOW(), + unique (handle, is_organization) +); + +create table app.accounts ( + id uuid + primary key + references auth.users (id), + handle app.valid_name not null unique, + is_organization boolean + generated always as (false) stored, + avatar_id uuid + references storage.objects (id), + display_name text + check (length(display_name) <= 128), + bio text check (length(bio) <= 512), + contact_email app.email_address, + created_at timestamp with time zone + not null + default NOW(), + constraint "fk_handle_registry" + foreign key + (handle, + is_organization) + references app.handle_registry (handle, + is_organization) +); + +create or replace function app.register_account() +returns trigger +language plpgsql +security definer +as $function$ +begin + insert into app.handle_registry (handle, is_organization) + values ( + new.raw_user_meta_data ->> 'handle', + false + ); + + insert into app.accounts (id, handle, display_name, bio, contact_email) + values ( + new.id, + new.raw_user_meta_data ->> 'handle', + new.raw_user_meta_data ->> 'display_name', + new.raw_user_meta_data ->> 'bio', + new.raw_user_meta_data ->> 'contact_email' + ); + return new; + end; +$function$; + +create or replace trigger on_auth_user_created +after insert +on auth.users +for each row +execute function app.register_account(); + +create table app.organizations ( + id uuid + primary key + default gen_random_uuid(), + handle app.valid_name not null unique, + is_organization boolean + generated always as (true) stored, + avatar_id uuid + references storage.objects (id), + display_name text + check (length(display_name) <= 128), + bio text check (length(bio) <= 512), + contact_email app.email_address, + created_at timestamp with time zone + not null + default NOW(), + constraint "fk_handle_registry" + foreign key + (handle, + is_organization) + references app.handle_registry (handle, + is_organization) +); + +create type app.membership_role as enum ('maintainer'); + +create table app.members ( + id uuid + primary key + default uuid_generate_v4(), + organization_id uuid + not null + references app.organizations (id), + account_id uuid + not null + references app.accounts (id), + role app.membership_role not null, + created_at timestamp with time zone + not null + default NOW(), + unique (organization_id, account_id) +); + +create or replace function app.register_organization_creator_as_member() +returns trigger +language plpgsql +security definer +as $function$ +begin + insert into app.members(organization_id, account_id, role) + values (new.id, auth.uid(), 'maintainer'); + + return new; + end; +$function$; + +create or replace trigger on_app_organization_created +after insert +on app.organizations +for each row +execute function app.register_organization_creator_as_member(); + +create or replace function app.update_avatar_id() +returns trigger +language plpgsql +security definer +as $function$ +declare + v_handle app.valid_name; + v_affected_account app.accounts := null; + begin + select (string_to_array(new.name, '-'::text))[1]::app.valid_name into v_handle; + + update app.accounts + set avatar_id = new.id + where handle = v_handle + returning * into v_affected_account; + + if not v_affected_account is null then + update auth.users u + set + "raw_user_meta_data" = u.raw_user_meta_data || jsonb_build_object( + 'avatar_path', new.name + ) + where u.id = v_affected_account.id; + else + update app.organizations + set avatar_id = new.id + where handle = v_handle; + end if; + + return new; + end; +$function$; + +create or replace trigger on_storage_object_created +after insert +on storage.objects +for each row +when (new.bucket_id = 'avatars') +execute function app.update_avatar_id(); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142137_package_tables_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142137_package_tables_100.snap new file mode 100644 index 000000000..2430b419a --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142137_package_tables_100.snap @@ -0,0 +1,104 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117142137_package_tables.sql +--- +insert into storage.buckets (id, name) +values + ('package_versions', 'package_versions'), + ('package_upgrades', 'package_upgrades'); + +create function app.to_package_name(handle app.valid_name, partial_name app.valid_name) +returns text +language sql +immutable +as $function$ +select format('%s-%s', $1, $2) +$function$; + +create table app.packages ( + id uuid primary key default gen_random_uuid(), + package_name text + not null + generated always as (app.to_package_name(handle, partial_name)) stored, + handle app.valid_name + not null + references app.handle_registry (handle), + partial_name app.valid_name not null, + control_description varchar(1000), + control_relocatable boolean not null default false, + control_requires varchar(128)[] + default cast('{}' as varchar(128)[]), + created_at timestamp with time zone + not null + default NOW(), + unique (handle, partial_name) +); + +create index "packages_partial_name_search_idx" +on app.packages +using gin +( + partial_name extensions.gin_trgm_ops +); + +create index "packages_handle_search_idx" +on app.packages +using gin +( + handle extensions.gin_trgm_ops +); + +create table app.package_versions ( + id uuid primary key default gen_random_uuid(), + package_id uuid + not null + references app.packages (id), + version_struct app.semver not null, + version text + not null + generated always as (app.semver_to_text(version_struct)) stored, + sql varchar(250000), + description_md varchar(250000), + created_at timestamp with time zone + not null + default NOW(), + unique (package_id, version_struct) +); + +create table app.package_upgrades ( + id uuid primary key default gen_random_uuid(), + package_id uuid + not null + references app.packages (id), + from_version_struct app.semver not null, + from_version text + not null + generated always as (app.semver_to_text(from_version_struct)) stored, + to_version_struct app.semver not null, + to_version text + not null + generated always as (app.semver_to_text(to_version_struct)) stored, + sql varchar(250000), + created_at timestamp with time zone + not null + default NOW(), + unique (package_id, + from_version_struct, + to_version_struct) +); + +create function app.version_text_to_handle(version text) +returns app.valid_name +language sql +immutable +as $function$ +select split_part($1, '-', 1) +$function$; + +create function app.version_text_to_package_partial_name(version text) +returns app.valid_name +language sql +immutable +as $function$ +select split_part($1, '--', 2) +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142137_package_tables_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142137_package_tables_80.snap new file mode 100644 index 000000000..d11509aaf --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142137_package_tables_80.snap @@ -0,0 +1,119 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117142137_package_tables.sql +--- +insert into storage.buckets (id, name) +values + ('package_versions', 'package_versions'), + ('package_upgrades', 'package_upgrades'); + +create function +app.to_package_name( + handle app.valid_name, + partial_name app.valid_name +) +returns text +language sql +immutable +as $function$ +select format('%s-%s', $1, $2) +$function$; + +create table app.packages ( + id uuid + primary key + default gen_random_uuid(), + package_name text + not null + generated always as (app.to_package_name( + handle, + partial_name + )) stored, + handle app.valid_name + not null + references app.handle_registry (handle), + partial_name app.valid_name not null, + control_description varchar(1000), + control_relocatable boolean + not null + default false, + control_requires varchar(128)[] + default cast('{}' as varchar(128)[]), + created_at timestamp with time zone + not null + default NOW(), + unique (handle, partial_name) +); + +create index "packages_partial_name_search_idx" +on app.packages +using gin +( + partial_name extensions.gin_trgm_ops +); + +create index "packages_handle_search_idx" +on app.packages +using gin +( + handle extensions.gin_trgm_ops +); + +create table app.package_versions ( + id uuid + primary key + default gen_random_uuid(), + package_id uuid + not null + references app.packages (id), + version_struct app.semver not null, + version text + not null + generated always as (app.semver_to_text(version_struct)) stored, + sql varchar(250000), + description_md varchar(250000), + created_at timestamp with time zone + not null + default NOW(), + unique (package_id, version_struct) +); + +create table app.package_upgrades ( + id uuid + primary key + default gen_random_uuid(), + package_id uuid + not null + references app.packages (id), + from_version_struct app.semver not null, + from_version text + not null + generated always as (app.semver_to_text(from_version_struct)) stored, + to_version_struct app.semver not null, + to_version text + not null + generated always as (app.semver_to_text(to_version_struct)) stored, + sql varchar(250000), + created_at timestamp with time zone + not null + default NOW(), + unique (package_id, + from_version_struct, + to_version_struct) +); + +create function app.version_text_to_handle(version text) +returns app.valid_name +language sql +immutable +as $function$ +select split_part($1, '-', 1) +$function$; + +create function app.version_text_to_package_partial_name(version text) +returns app.valid_name +language sql +immutable +as $function$ +select split_part($1, '--', 2) +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142138_developer_tools_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142138_developer_tools_100.snap new file mode 100644 index 000000000..0964d7649 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142138_developer_tools_100.snap @@ -0,0 +1,32 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117142138_developer_tools.sql +--- +create or replace function app.simulate_login(email citext) +returns void +language sql +as $function$ +/* + Simulated JWT of logged in user + */ + + select + set_config( + 'request.jwt.claims', + ( + select + json_build_object( + 'sub', + id, + 'role', + 'authenticated' + )::text + from + auth.users + where + email = $1 + ), + true + ), + set_config('role', 'authenticated', true) +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142138_developer_tools_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142138_developer_tools_80.snap new file mode 100644 index 000000000..0964d7649 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142138_developer_tools_80.snap @@ -0,0 +1,32 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117142138_developer_tools.sql +--- +create or replace function app.simulate_login(email citext) +returns void +language sql +as $function$ +/* + Simulated JWT of logged in user + */ + + select + set_config( + 'request.jwt.claims', + ( + select + json_build_object( + 'sub', + id, + 'role', + 'authenticated' + )::text + from + auth.users + where + email = $1 + ), + true + ), + set_config('role', 'authenticated', true) +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142141_security_utilities_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142141_security_utilities_100.snap new file mode 100644 index 000000000..264efaf5a --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142141_security_utilities_100.snap @@ -0,0 +1,98 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117142141_security_utilities.sql +--- +create function app.is_organization_maintainer(account_id uuid, organization_id uuid) +returns boolean +language sql +stable +as $function$ +-- Does the currently authenticated user have permission to admin orgs and org members? + select + exists( + select + 1 + from + app.members m + where + m.account_id = $1 + and m.organization_id = $2 + and m.role = 'maintainer' + ) +$function$; + +create function app.is_handle_maintainer(account_id uuid, handle app.valid_name) +returns boolean +language sql +stable +as $function$ +select + exists( + select + 1 + from + app.accounts acc + where + acc.id = $1 + and acc.handle = $2 + ) + or exists( + select + 1 + from + app.organizations o + join app.members m + on o.id = m.organization_id + where + m.role = 'maintainer' + and m.account_id = $1 + and o.handle = $2 + ) +$function$; + +create function app.is_package_maintainer(account_id uuid, package_id uuid) +returns boolean +language sql +stable +as $function$ +select + exists( + select + 1 + from + app.accounts acc + join app.packages p + on acc.handle = p.handle + where + acc.id = $1 + and p.id = $2 + ) + or exists( + -- current user is maintainer of org that owns the package + select + 1 + from + app.packages p + join app.organizations o + on p.handle = o.handle + join app.members m + on o.id = m.organization_id + where + m.role = 'maintainer' + and m.account_id = $1 + and p.id = $2 + ) +$function$; + +create function app.is_package_version_maintainer(account_id uuid, package_version_id uuid) +returns boolean +language sql +stable +as $function$ +select + app.is_package_maintainer($1, pv.package_id) + from + app.package_versions pv + where + pv.id = $2 +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142141_security_utilities_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142141_security_utilities_80.snap new file mode 100644 index 000000000..a35e88350 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142141_security_utilities_80.snap @@ -0,0 +1,106 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117142141_security_utilities.sql +--- +create function +app.is_organization_maintainer( + account_id uuid, + organization_id uuid +) +returns boolean +language sql +stable +as $function$ +-- Does the currently authenticated user have permission to admin orgs and org members? + select + exists( + select + 1 + from + app.members m + where + m.account_id = $1 + and m.organization_id = $2 + and m.role = 'maintainer' + ) +$function$; + +create function app.is_handle_maintainer(account_id uuid, handle app.valid_name) +returns boolean +language sql +stable +as $function$ +select + exists( + select + 1 + from + app.accounts acc + where + acc.id = $1 + and acc.handle = $2 + ) + or exists( + select + 1 + from + app.organizations o + join app.members m + on o.id = m.organization_id + where + m.role = 'maintainer' + and m.account_id = $1 + and o.handle = $2 + ) +$function$; + +create function app.is_package_maintainer(account_id uuid, package_id uuid) +returns boolean +language sql +stable +as $function$ +select + exists( + select + 1 + from + app.accounts acc + join app.packages p + on acc.handle = p.handle + where + acc.id = $1 + and p.id = $2 + ) + or exists( + -- current user is maintainer of org that owns the package + select + 1 + from + app.packages p + join app.organizations o + on p.handle = o.handle + join app.members m + on o.id = m.organization_id + where + m.role = 'maintainer' + and m.account_id = $1 + and p.id = $2 + ) +$function$; + +create function +app.is_package_version_maintainer( + account_id uuid, + package_version_id uuid +) +returns boolean +language sql +stable +as $function$ +select + app.is_package_maintainer($1, pv.package_id) + from + app.package_versions pv + where + pv.id = $2 +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142142_security_definitions_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142142_security_definitions_100.snap new file mode 100644 index 000000000..fd4842bcc --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142142_security_definitions_100.snap @@ -0,0 +1,196 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117142142_security_definitions.sql +--- +grant INSERT (handle, is_organization) on table app.handle_registry to authenticated; + +alter table app.accounts + enable row level security; + +grant UPDATE (avatar_id, display_name, bio, contact_email) on table app.accounts to authenticated; + +create policy accounts_update_policy +on app.accounts +as permissive +for update +to authenticated +using (id = auth.uid()); + +create policy accounts_select_policy +on app.accounts +as permissive +for select +to authenticated +using (true); + +alter table app.organizations + enable row level security; + +grant INSERT (handle, +avatar_id, +display_name, +bio, +contact_email) +on table app.organizations +to authenticated; + +grant UPDATE (avatar_id, +display_name, +bio, +contact_email) +on table app.organizations +to authenticated; + +create policy organizations_insert_policy +on app.organizations +as permissive +for insert +to authenticated +with check (true); + +create policy organizations_update_policy +on app.organizations +as permissive +for update +to authenticated +using (app.is_organization_maintainer(auth.uid(), id)); + +create policy organizations_select_policy +on app.organizations +as permissive +for select +to authenticated +using (true); + +alter table app.members + enable row level security; + +grant INSERT (organization_id, account_id, role) on table app.members to authenticated; + +grant DELETE on table app.members to authenticated; + +create policy members_insert_policy +on app.members +as permissive +for insert +to authenticated +with check (app.is_organization_maintainer( + auth.uid(), + organization_id +)); + +create policy members_delete_policy +on app.members +as permissive +for delete +to authenticated +using (app.is_organization_maintainer( + auth.uid(), + organization_id +)); + +create policy members_select_policy +on app.members +as permissive +for select +to authenticated +using (true); + +alter table app.packages + enable row level security; + +grant INSERT (partial_name, handle) on table app.packages to authenticated; + +create policy package_insert_policy +on app.packages +as permissive +for insert +to authenticated +with check (app.is_handle_maintainer(auth.uid(), handle)); + +create policy packages_select_policy +on app.packages +as permissive +for select +to authenticated +using (true); + +alter table app.package_versions + enable row level security; + +grant INSERT (package_id, +version_struct, +sql, +description_md) +on table app.package_versions +to authenticated; + +create policy package_versions_insert_policy +on app.package_versions +as permissive +for insert +to authenticated +with check (app.is_package_maintainer(auth.uid(), package_id)); + +create policy package_versions_update_policy +on app.package_versions +as permissive +for update +to authenticated +using (app.is_package_maintainer(auth.uid(), package_id)); + +create policy package_versions_select_policy +on app.package_versions +as permissive +for select +to public +using (true); + +alter table app.package_upgrades + enable row level security; + +grant INSERT (package_id, +from_version_struct, +to_version_struct, +sql) +on table app.package_upgrades +to authenticated; + +create policy package_upgrades_insert_policy +on app.package_upgrades +as permissive +for insert +to authenticated +with check (app.is_package_maintainer(auth.uid(), package_id)); + +create policy package_upgrades_update_policy +on app.package_upgrades +as permissive +for update +to authenticated +using (app.is_package_maintainer(auth.uid(), package_id)); + +create policy package_upgrades_select_policy +on app.package_upgrades +as permissive +for select +to public +using (true); + +create policy storage_objects_insert_policy +on storage.objects +as permissive +for insert +to authenticated +with check (app.is_handle_maintainer( + auth.uid(), + cast((string_to_array(name, cast('-' as text)))[1] + as app.valid_name) +)); + +create policy storage_objects_select_policy +on storage.objects +as permissive +for select +to public +using (bucket_id = 'avatars'); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142142_security_definitions_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142142_security_definitions_80.snap new file mode 100644 index 000000000..39a2f7a1d --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117142142_security_definitions_80.snap @@ -0,0 +1,226 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117142142_security_definitions.sql +--- +grant INSERT (handle, +is_organization) +on table app.handle_registry +to authenticated; + +alter table app.accounts + enable row level security; + +grant UPDATE (avatar_id, +display_name, +bio, +contact_email) +on table app.accounts +to authenticated; + +create policy accounts_update_policy +on app.accounts +as permissive +for update +to authenticated +using (id = auth.uid()); + +create policy accounts_select_policy +on app.accounts +as permissive +for select +to authenticated +using (true); + +alter table app.organizations + enable row level security; + +grant INSERT (handle, +avatar_id, +display_name, +bio, +contact_email) +on table app.organizations +to authenticated; + +grant UPDATE (avatar_id, +display_name, +bio, +contact_email) +on table app.organizations +to authenticated; + +create policy organizations_insert_policy +on app.organizations +as permissive +for insert +to authenticated +with check (true); + +create policy organizations_update_policy +on app.organizations +as permissive +for update +to authenticated +using (app.is_organization_maintainer( + auth.uid(), + id +)); + +create policy organizations_select_policy +on app.organizations +as permissive +for select +to authenticated +using (true); + +alter table app.members + enable row level security; + +grant INSERT (organization_id, +account_id, +role) +on table app.members +to authenticated; + +grant DELETE on table app.members to authenticated; + +create policy members_insert_policy +on app.members +as permissive +for insert +to authenticated +with check (app.is_organization_maintainer( + auth.uid(), + organization_id +)); + +create policy members_delete_policy +on app.members +as permissive +for delete +to authenticated +using (app.is_organization_maintainer( + auth.uid(), + organization_id +)); + +create policy members_select_policy +on app.members +as permissive +for select +to authenticated +using (true); + +alter table app.packages + enable row level security; + +grant INSERT (partial_name, handle) on table app.packages to authenticated; + +create policy package_insert_policy +on app.packages +as permissive +for insert +to authenticated +with check (app.is_handle_maintainer( + auth.uid(), + handle +)); + +create policy packages_select_policy +on app.packages +as permissive +for select +to authenticated +using (true); + +alter table app.package_versions + enable row level security; + +grant INSERT (package_id, +version_struct, +sql, +description_md) +on table app.package_versions +to authenticated; + +create policy package_versions_insert_policy +on app.package_versions +as permissive +for insert +to authenticated +with check (app.is_package_maintainer( + auth.uid(), + package_id +)); + +create policy package_versions_update_policy +on app.package_versions +as permissive +for update +to authenticated +using (app.is_package_maintainer( + auth.uid(), + package_id +)); + +create policy package_versions_select_policy +on app.package_versions +as permissive +for select +to public +using (true); + +alter table app.package_upgrades + enable row level security; + +grant INSERT (package_id, +from_version_struct, +to_version_struct, +sql) +on table app.package_upgrades +to authenticated; + +create policy package_upgrades_insert_policy +on app.package_upgrades +as permissive +for insert +to authenticated +with check (app.is_package_maintainer( + auth.uid(), + package_id +)); + +create policy package_upgrades_update_policy +on app.package_upgrades +as permissive +for update +to authenticated +using (app.is_package_maintainer( + auth.uid(), + package_id +)); + +create policy package_upgrades_select_policy +on app.package_upgrades +as permissive +for select +to public +using (true); + +create policy storage_objects_insert_policy +on storage.objects +as permissive +for insert +to authenticated +with check (app.is_handle_maintainer( + auth.uid(), + cast((string_to_array(name, cast('-' as text)))[1] + as app.valid_name) +)); + +create policy storage_objects_select_policy +on storage.objects +as permissive +for select +to public +using (bucket_id = 'avatars'); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155720_views_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155720_views_100.snap new file mode 100644 index 000000000..6835dfec5 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155720_views_100.snap @@ -0,0 +1,99 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117155720_views.sql +--- +create view public.accounts +as select + acc.id, + acc.handle, + obj.name as avatar_path, + acc.display_name, + acc.bio, + acc.contact_email, + acc.created_at +from + app.accounts as acc + left outer join + storage.objects as obj + on acc.avatar_id = obj.id; + +create view public.organizations +as select + org.id, + org.handle, + obj.name as avatar_path, + org.display_name, + org.bio, + org.contact_email, + org.created_at +from + app.organizations as org + left outer join + storage.objects as obj + on org.avatar_id = obj.id; + +create view public.members +as select + aio.organization_id, + aio.account_id, + aio.role, + aio.created_at +from + app.members as aio; + +create view public.packages +as select + pa.id, + pa.package_name, + pa.handle, + pa.partial_name, + newest_ver.version as latest_version, + newest_ver.description_md, + pa.control_description, + pa.control_requires, + pa.created_at +from + app.packages as pa, + lateral ( + select + * + from + app.package_versions as pv + where + pv.package_id = pa.id + order by pv.version_struct + limit 1 + ) + as newest_ver; + +create view public.package_versions +as select + pv.id, + pv.package_id, + pa.package_name, + pv.version, + pv.sql, + pv.description_md, + pa.control_description, + pa.control_requires, + pv.created_at +from + app.packages as pa + inner join + app.package_versions as pv + on pa.id = pv.package_id; + +create view public.package_upgrades +as select + pu.id, + pu.package_id, + pa.package_name, + pu.from_version, + pu.to_version, + pu.sql, + pu.created_at +from + app.packages as pa + inner join + app.package_upgrades as pu + on pa.id = pu.package_id; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155720_views_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155720_views_80.snap new file mode 100644 index 000000000..6835dfec5 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155720_views_80.snap @@ -0,0 +1,99 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117155720_views.sql +--- +create view public.accounts +as select + acc.id, + acc.handle, + obj.name as avatar_path, + acc.display_name, + acc.bio, + acc.contact_email, + acc.created_at +from + app.accounts as acc + left outer join + storage.objects as obj + on acc.avatar_id = obj.id; + +create view public.organizations +as select + org.id, + org.handle, + obj.name as avatar_path, + org.display_name, + org.bio, + org.contact_email, + org.created_at +from + app.organizations as org + left outer join + storage.objects as obj + on org.avatar_id = obj.id; + +create view public.members +as select + aio.organization_id, + aio.account_id, + aio.role, + aio.created_at +from + app.members as aio; + +create view public.packages +as select + pa.id, + pa.package_name, + pa.handle, + pa.partial_name, + newest_ver.version as latest_version, + newest_ver.description_md, + pa.control_description, + pa.control_requires, + pa.created_at +from + app.packages as pa, + lateral ( + select + * + from + app.package_versions as pv + where + pv.package_id = pa.id + order by pv.version_struct + limit 1 + ) + as newest_ver; + +create view public.package_versions +as select + pv.id, + pv.package_id, + pa.package_name, + pv.version, + pv.sql, + pv.description_md, + pa.control_description, + pa.control_requires, + pv.created_at +from + app.packages as pa + inner join + app.package_versions as pv + on pa.id = pv.package_id; + +create view public.package_upgrades +as select + pu.id, + pu.package_id, + pa.package_name, + pu.from_version, + pu.to_version, + pu.sql, + pu.created_at +from + app.packages as pa + inner join + app.package_upgrades as pu + on pa.id = pu.package_id; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155820_rpc_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155820_rpc_100.snap new file mode 100644 index 000000000..821d10d8a --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155820_rpc_100.snap @@ -0,0 +1,23 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117155820_rpc.sql +--- +create or replace function +public.search_packages( + handle citext default null, + partial_name citext default null +) +returns setof public.packages +language sql +stable +as $function$ +select * + from public.packages + where + ($1 is null or handle <% $1 or handle ~ $1) + and + ($2 is null or partial_name <% $2 or partial_name ~ $2) + order by + coalesce(extensions.similarity($1, handle), 0) + coalesce(extensions.similarity($2, partial_name), 0) desc, + created_at desc; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155820_rpc_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155820_rpc_80.snap new file mode 100644 index 000000000..821d10d8a --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20220117155820_rpc_80.snap @@ -0,0 +1,23 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20220117155820_rpc.sql +--- +create or replace function +public.search_packages( + handle citext default null, + partial_name citext default null +) +returns setof public.packages +language sql +stable +as $function$ +select * + from public.packages + where + ($1 is null or handle <% $1 or handle ~ $1) + and + ($2 is null or partial_name <% $2 or partial_name ~ $2) + order by + coalesce(extensions.similarity($1, handle), 0) + coalesce(extensions.similarity($2, partial_name), 0) desc, + created_at desc; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230323180034_reserved_user_accts_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230323180034_reserved_user_accts_100.snap new file mode 100644 index 000000000..7d9ff0b71 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230323180034_reserved_user_accts_100.snap @@ -0,0 +1,145 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230323180034_reserved_user_accts.sql +--- +insert into auth.users ( + instance_id, + id, + aud, + role, + email, + encrypted_password, + email_confirmed_at, + raw_app_meta_data, + raw_user_meta_data, + is_sso_user +) +values + ( + '00000000-0000-0000-0000-000000000000', + gen_random_uuid(), + 'authenticated', + 'authenticated', + 'oliver@oliverrice.com', + 'TBD', + NOW(), + '{"provider": "email", "providers": ["email"]}', + '{"handle": "olirice", "display_name": "Oli", "bio": "Supabase Staff"}', + false + ), + ( + '00000000-0000-0000-0000-000000000000', + gen_random_uuid(), + 'authenticated', + 'authenticated', + 'alaister@supabase.io', + 'TBD', + NOW(), + '{"provider": "email", "providers": ["email"]}', + '{"handle": "alaister", "display_name": "Alaister", "bio": "Supabase Staff"}', + false + ), + ( + '00000000-0000-0000-0000-000000000000', + gen_random_uuid(), + 'authenticated', + 'authenticated', + 'copple@supabase.io', + 'TBD', + NOW(), + '{"provider": "email", "providers": ["email"]}', + '{"handle": "kiwicopple", "display_name": "Copple", "bio": "Supabase Staff"}', + false + ), + ( + '00000000-0000-0000-0000-000000000000', + gen_random_uuid(), + 'authenticated', + 'authenticated', + 'michel@supabase.io', + 'TBD', + NOW(), + '{"provider": "email", "providers": ["email"]}', + '{"handle": "michelp", "display_name": "Michele", "bio": "Supabase Staff"}', + false + ), + ( + '00000000-0000-0000-0000-000000000000', + gen_random_uuid(), + 'authenticated', + 'authenticated', + 'mark@supabase.io', + 'TBD', + NOW(), + '{"provider": "email", "providers": ["email"]}', + '{"handle": "burggraf", "display_name": "Mark", "bio": "Supabase Staff"}', + false + ); + +insert into app.handle_registry (handle, is_organization) +values + ('supabase', true), + ('langchain', true), + ('admin', false), + ('administrator', false), + ('superuser', false), + ('superadmin', false), + ('root', false), + ('user', false), + ('guest', false), + ('anon', false), + ('authenticated', false), + ('sysadmin', false), + ('support', false), + ('manager', false), + ('default', false), + ('staff', false), + ('help', false), + ('helpdesk', false), + ('test', false), + ('password', false), + ('demo', false), + ('service', false), + ('info', false), + ('webmaster', false), + ('security', false), + ('installer', false); + +begin; + +select app.simulate_login('oliver@oliverrice.com'); + +insert into app.organizations (handle, display_name, bio) +values + ( + 'supabase', + 'Supabase', + 'Build in a weekend, scale to millions' + ); + +commit; + +insert into app.members (organization_id, account_id, role) +select + o.id, + acc.id, + 'maintainer' +from + app.organizations as o, + app.accounts as acc +where + acc.handle <> 'olirice'; + +begin; + +select app.simulate_login('oliver@oliverrice.com'); + +insert into app.organizations (handle, display_name, bio) +values + ( + 'langchain', + 'LangChain', + 'LangChain is a framework for developing applications powered by language models' + ); + +commit; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230323180034_reserved_user_accts_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230323180034_reserved_user_accts_80.snap new file mode 100644 index 000000000..7d9ff0b71 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230323180034_reserved_user_accts_80.snap @@ -0,0 +1,145 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230323180034_reserved_user_accts.sql +--- +insert into auth.users ( + instance_id, + id, + aud, + role, + email, + encrypted_password, + email_confirmed_at, + raw_app_meta_data, + raw_user_meta_data, + is_sso_user +) +values + ( + '00000000-0000-0000-0000-000000000000', + gen_random_uuid(), + 'authenticated', + 'authenticated', + 'oliver@oliverrice.com', + 'TBD', + NOW(), + '{"provider": "email", "providers": ["email"]}', + '{"handle": "olirice", "display_name": "Oli", "bio": "Supabase Staff"}', + false + ), + ( + '00000000-0000-0000-0000-000000000000', + gen_random_uuid(), + 'authenticated', + 'authenticated', + 'alaister@supabase.io', + 'TBD', + NOW(), + '{"provider": "email", "providers": ["email"]}', + '{"handle": "alaister", "display_name": "Alaister", "bio": "Supabase Staff"}', + false + ), + ( + '00000000-0000-0000-0000-000000000000', + gen_random_uuid(), + 'authenticated', + 'authenticated', + 'copple@supabase.io', + 'TBD', + NOW(), + '{"provider": "email", "providers": ["email"]}', + '{"handle": "kiwicopple", "display_name": "Copple", "bio": "Supabase Staff"}', + false + ), + ( + '00000000-0000-0000-0000-000000000000', + gen_random_uuid(), + 'authenticated', + 'authenticated', + 'michel@supabase.io', + 'TBD', + NOW(), + '{"provider": "email", "providers": ["email"]}', + '{"handle": "michelp", "display_name": "Michele", "bio": "Supabase Staff"}', + false + ), + ( + '00000000-0000-0000-0000-000000000000', + gen_random_uuid(), + 'authenticated', + 'authenticated', + 'mark@supabase.io', + 'TBD', + NOW(), + '{"provider": "email", "providers": ["email"]}', + '{"handle": "burggraf", "display_name": "Mark", "bio": "Supabase Staff"}', + false + ); + +insert into app.handle_registry (handle, is_organization) +values + ('supabase', true), + ('langchain', true), + ('admin', false), + ('administrator', false), + ('superuser', false), + ('superadmin', false), + ('root', false), + ('user', false), + ('guest', false), + ('anon', false), + ('authenticated', false), + ('sysadmin', false), + ('support', false), + ('manager', false), + ('default', false), + ('staff', false), + ('help', false), + ('helpdesk', false), + ('test', false), + ('password', false), + ('demo', false), + ('service', false), + ('info', false), + ('webmaster', false), + ('security', false), + ('installer', false); + +begin; + +select app.simulate_login('oliver@oliverrice.com'); + +insert into app.organizations (handle, display_name, bio) +values + ( + 'supabase', + 'Supabase', + 'Build in a weekend, scale to millions' + ); + +commit; + +insert into app.members (organization_id, account_id, role) +select + o.id, + acc.id, + 'maintainer' +from + app.organizations as o, + app.accounts as acc +where + acc.handle <> 'olirice'; + +begin; + +select app.simulate_login('oliver@oliverrice.com'); + +insert into app.organizations (handle, display_name, bio) +values + ( + 'langchain', + 'LangChain', + 'LangChain is a framework for developing applications powered by language models' + ); + +commit; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230328185043_olirice_asciiplot_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230328185043_olirice_asciiplot_100.snap new file mode 100644 index 000000000..c6cddcfbf --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230328185043_olirice_asciiplot_100.snap @@ -0,0 +1,191 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230328185043_olirice_asciiplot.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'olirice', + 'asciiplot', + 'A Toy ASCII Plotting Library', + false, + '{}' + ); + +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_name = 'olirice-asciiplot' + ), + (0, 0, 1), + ' +CREATE TYPE scatter_state AS ( + x_arr NUMERIC[], + y_arr NUMERIC[], + title TEXT, + height INTEGER, + width INTEGER +); + +CREATE OR REPLACE FUNCTION scatter_sfunc( + state scatter_state, + x numeric, + y numeric, + title TEXT, + height INTEGER, + width INTEGER +) +RETURNS scatter_state +LANGUAGE plpgsql +AS $$ +BEGIN + state.x_arr := array_append(coalesce(state.x_arr, array[]::numeric[]), x); + state.y_arr := array_append(coalesce(state.y_arr, array[]::numeric[]), y); + state.title := coalesce(state.title, title); + state.height := coalesce(state.height, height); + state.width := coalesce(state.width, width); + RETURN state; +END; +$$; + + +CREATE OR REPLACE FUNCTION scatter_internal( + state scatter_state +) +RETURNS TEXT +LANGUAGE plpgsql +AS $$ +DECLARE + plot text[] := array[]::text[]; + + i int := 0; + j int := 0; + max_x numeric := max(v) FROM unnest(state.x_arr) arr(v); + max_y numeric := max(v) FROM unnest(state.y_arr) arr(v); + min_x numeric := min(v) FROM unnest(state.x_arr) arr(v); + min_y numeric := min(v) FROM unnest(state.y_arr) arr(v); + x_range numeric := abs(max_x - min_x); + y_range numeric := abs(max_y - min_y); + + x_scale numeric := x_range / (state.width); + y_scale numeric := y_range / (state.height - 2); + point_idx int; + header text; +begin + for i in 1..(state.height - 2) loop + plot := array_append(plot, repeat('' '', state.width)); + end loop; + + for point_idx in 1..array_length(state.x_arr, 1) loop + i := round((state.y_arr[point_idx] - min_y) / y_scale)::integer; + j := round((state.x_arr[point_idx] - min_x) / x_scale)::integer; + + plot[i] := overlay(plot[i] placing ''*'' from j for 1); + end loop; + + header = ( + repeat('' '', (state.width - character_length(state.title)) / 2) + || state.title + || repeat('' '', (state.width - character_length(state.title)) / 2) + ); + header = header || E''\n'' || repeat(''-'', state.width) || E''\n''; + + return + header || string_agg(v_elem, E''\n'' order by ix desc) + from + unnest(plot) with ordinality v_arr(v_elem, ix); +end; +$$; + + +CREATE AGGREGATE scatter( + x NUMERIC, + y NUMERIC, + title TEXT, + height INTEGER, + width INTEGER +) ( + STYPE = scatter_state, + SFUNC = scatter_sfunc, + FINALFUNC = scatter_internal +); + +comment on type scatter_state is e''internal''; + +', + ' +# asciiplot + +asciiplot is a toy library for producing ASCII scatterplots from PostgreSQL queries. +Please note that it is not indended for serious use. + +### Usage + +```sql +select + scatter( + val::numeric, --x + val::numeric, -- y + ''stonks!'', -- title + 15, -- height + 50 --width + ) +from + generate_series(1,10) vals(val) + +/* + stonks! +---------------------------------------------- +| * +| +| * +| * +| +| * +| +| * +| * +| +| * +| +| * +| * +*/ +``` +' + ); + +insert into app.package_upgrades ( + package_id, + from_version_struct, + to_version_struct, + sql +) +values + ( + ( + select + id + from + app.packages + where + package_name = 'olirice-asciiplot' + ), + (0, 0, 1), + (0, 0, 2), + ' +comment on type scatter_state is e''internal''; +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230328185043_olirice_asciiplot_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230328185043_olirice_asciiplot_80.snap new file mode 100644 index 000000000..4b92ffdcb --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230328185043_olirice_asciiplot_80.snap @@ -0,0 +1,196 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230328185043_olirice_asciiplot.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'olirice', + 'asciiplot', + 'A Toy ASCII Plotting Library', + false, + '{}' + ); + +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_name = 'olirice-asciiplot' + ), + (0, 0, 1), + ' +CREATE TYPE scatter_state AS ( + x_arr NUMERIC[], + y_arr NUMERIC[], + title TEXT, + height INTEGER, + width INTEGER +); + +CREATE OR REPLACE FUNCTION scatter_sfunc( + state scatter_state, + x numeric, + y numeric, + title TEXT, + height INTEGER, + width INTEGER +) +RETURNS scatter_state +LANGUAGE plpgsql +AS $$ +BEGIN + state.x_arr := array_append(coalesce(state.x_arr, array[]::numeric[]), x); + state.y_arr := array_append(coalesce(state.y_arr, array[]::numeric[]), y); + state.title := coalesce(state.title, title); + state.height := coalesce(state.height, height); + state.width := coalesce(state.width, width); + RETURN state; +END; +$$; + + +CREATE OR REPLACE FUNCTION scatter_internal( + state scatter_state +) +RETURNS TEXT +LANGUAGE plpgsql +AS $$ +DECLARE + plot text[] := array[]::text[]; + + i int := 0; + j int := 0; + max_x numeric := max(v) FROM unnest(state.x_arr) arr(v); + max_y numeric := max(v) FROM unnest(state.y_arr) arr(v); + min_x numeric := min(v) FROM unnest(state.x_arr) arr(v); + min_y numeric := min(v) FROM unnest(state.y_arr) arr(v); + x_range numeric := abs(max_x - min_x); + y_range numeric := abs(max_y - min_y); + + x_scale numeric := x_range / (state.width); + y_scale numeric := y_range / (state.height - 2); + point_idx int; + header text; +begin + for i in 1..(state.height - 2) loop + plot := array_append(plot, repeat('' '', state.width)); + end loop; + + for point_idx in 1..array_length(state.x_arr, 1) loop + i := round((state.y_arr[point_idx] - min_y) / y_scale)::integer; + j := round((state.x_arr[point_idx] - min_x) / x_scale)::integer; + + plot[i] := overlay(plot[i] placing ''*'' from j for 1); + end loop; + + header = ( + repeat('' '', (state.width - character_length(state.title)) / 2) + || state.title + || repeat('' '', (state.width - character_length(state.title)) / 2) + ); + header = header || E''\n'' || repeat(''-'', state.width) || E''\n''; + + return + header || string_agg(v_elem, E''\n'' order by ix desc) + from + unnest(plot) with ordinality v_arr(v_elem, ix); +end; +$$; + + +CREATE AGGREGATE scatter( + x NUMERIC, + y NUMERIC, + title TEXT, + height INTEGER, + width INTEGER +) ( + STYPE = scatter_state, + SFUNC = scatter_sfunc, + FINALFUNC = scatter_internal +); + +comment on type scatter_state is e''internal''; + +', + ' +# asciiplot + +asciiplot is a toy library for producing ASCII scatterplots from PostgreSQL queries. +Please note that it is not indended for serious use. + +### Usage + +```sql +select + scatter( + val::numeric, --x + val::numeric, -- y + ''stonks!'', -- title + 15, -- height + 50 --width + ) +from + generate_series(1,10) vals(val) + +/* + stonks! +---------------------------------------------- +| * +| +| * +| * +| +| * +| +| * +| * +| +| * +| +| * +| * +*/ +``` +' + ); + +insert into app.package_upgrades ( + package_id, + from_version_struct, + to_version_struct, + sql +) +values + ( + ( + select + id + from + app.packages + where + package_name = 'olirice-asciiplot' + ), + (0, 0, 1), + (0, 0, 2), + ' +comment on type scatter_state is e''internal''; +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230330155137_supabase_dbdev_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230330155137_supabase_dbdev_100.snap new file mode 100644 index 000000000..6d46331e8 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230330155137_supabase_dbdev_100.snap @@ -0,0 +1,301 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230330155137_supabase_dbdev.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'supabase', + 'dbdev', + 'Install pacakges from the dbdev package index', + false, + '{}' + ); + +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_name = 'supabase-dbdev' + ), + (0, 0, 2), + ' + +create schema dbdev; + +create or replace function dbdev.install(package_name text) + returns bool + language plpgsql +as $$ +declare + -- Endpoint + base_url text = ''https://api.database.dev/rest/v1/''; + apikey text = ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyYndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzIsImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJrzM0AQKsu_5k134s''; + + http_ext_schema regnamespace = extnamespace::regnamespace from pg_catalog.pg_extension where extname = ''http'' limit 1; + pgtle_is_available bool = true from pg_catalog.pg_extension where extname = ''pg_tle'' limit 1; + -- HTTP respones + rec jsonb; + status int; + contents json; + + -- Install Record + rec_sql text; + rec_ver text; + rec_from_ver text; + rec_to_ver text; + rec_package_name text; + rec_description text; + rec_requires text[]; +begin + + if http_ext_schema is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the http extension and it is not available''); + end if; + + if pgtle_is_available is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the pgtle extension and it is not available''); + end if; + + ------------------- + -- Base Versions -- + ------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_versions?select=package_name,version,sql,control_description,control_requires&limit=50&package_name=eq.%s'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading versions from dbdev''); + end if; + + if contents is null or json_typeof(contents) <> ''array'' or json_array_length(contents) = 0 then + raise exception using errcode=''22000'', message=format(''No versions for package named named %s'', package_name); + end if; + + for rec_package_name, rec_ver, rec_sql, rec_description, rec_requires in select + (r ->> ''package_name''), + (r ->> ''version''), + (r ->> ''sql''), + (r ->> ''control_description''), + to_json(rec ->> ''control_requires'') #>> ''{}'' + from + json_array_elements(contents) as r + loop + + if not exists ( + select true + from pgtle.available_extension_versions() + where + -- TLE will not allow multiple full install scripts + -- TODO(OR) open upstream issue to discuss + name = rec_package_name + ) then + perform pgtle.install_extension(rec_package_name, rec_ver, rec_package_name, rec_sql); + end if; + end loop; + + ---------------------- + -- Upgrade Versions -- + ---------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_upgrades?select=package_name,from_version,to_version,sql&limit=50&package_name=eq.%s'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading upgrade pathes from dbdev''); + end if; + + if json_typeof(contents) <> ''array'' then + raise exception using errcode=''22000'', message=format(''Invalid response from dbdev upgrade pathes''); + end if; + + for rec_package_name, rec_from_ver, rec_to_ver, rec_sql in select + (r ->> ''package_name''), + (r ->> ''from_version''), + (r ->> ''to_version''), + (r ->> ''sql'') + from + json_array_elements(contents) as r + loop + + if not exists ( + select true + from pgtle.extension_update_paths(rec_package_name) + where + source = rec_from_ver + and target = rec_to_ver + and path is not null + ) then + perform pgtle.install_update_path(rec_package_name, rec_from_ver, rec_to_ver, rec_sql); + end if; + end loop; + + -------------------------- + -- Send Download Notice -- + -------------------------- + -- Notifies dbdev that a package has been downloaded and records IP + user agent so we can compute unique download counts + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''POST'', + format( + ''%srpc/register_download'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header, + (''x-client-info'', ''dbdev/0.0.2'')::http_header + ], + ''application/json'', + json_build_object(''package_name'', $stmt$ || pg_catalog.quote_literal($1) || $stmt$)::text + ) + ) x + limit 1; $stmt$ + into rec; + + return true; +end; +$$; + +', + ' +# dbdev + +dbdev is the SQL client for database.new and is the primary way end users interact with the package (pglet) registry. + +dbdev can be used to load packages from the registry. For example: + +```sql +-- Load the package from the package index +select dbdev.install(''olirice-index_advisor''); +``` +Where `olirice` is the handle of the author and `index_advisor` is the name of the pglet. + +Once installed, pglets are visible in PostgreSQL as extensions. At that point they can be enabled with standard Postgres commands i.e. the `create extension` + +To improve reproducibility, we recommend __always__ specifying the package version in your `create extension` statements. + +For example: +```sql +-- Enable the extension +create extension "olirice-index_advisor" + schema ''public'' + version ''0.1.0''; +``` + +Which creates all tables/indexes/functions/etc specified by the extension. + +## How to Install + +The in-database SQL client for the package registry is named `dbdev`. You can bootstrap the client with: + +```sql +/*--------------------- +---- install dbdev ---- +---------------------- +Requires: + - pg_tle: https://github.com/aws/pg_tle + - pgsql-http: https://github.com/pramsey/pgsql-http +*/ +create extension if not exists http with schema extensions; +create extension if not exists pg_tle; +select pgtle.uninstall_extension_if_exists(''supabase-dbdev''); +drop extension if exists "supabase-dbdev"; +select + pgtle.install_extension( + ''supabase-dbdev'', + resp.contents ->> ''version'', + ''PostgreSQL package manager'', + resp.contents ->> ''sql'' + ) +from http( + ( + ''GET'', + ''https://api.database.dev/rest/v1/'' + || ''package_versions?select=sql,version'' + || ''&package_name=eq.supabase-dbdev'' + || ''&order=version.desc'' + || ''&limit=1'', + array[ + ( + ''apiKey'', + ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJp'' + || ''c3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyY'' + || ''ndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzI'' + || ''sImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJ'' + || ''rzM0AQKsu_5k134s'' + )::http_header + ], + null, + null + ) +) x, +lateral ( + select + ((row_to_json(x) -> ''content'') #>> ''{}'')::json -> 0 +) resp(contents); +create extension "supabase-dbdev"; +select dbdev.install(''supabase-dbdev''); +drop extension if exists "supabase-dbdev"; +create extension "supabase-dbdev"; +``` + +With the client ready, search for packages on [database.dev](database.dev) and install them with + +```sql +select dbdev.install(''handle-package_name''); +create extension "handle-package_name" + schema ''public'' + version ''1.2.3''; +``` +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230330155137_supabase_dbdev_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230330155137_supabase_dbdev_80.snap new file mode 100644 index 000000000..9bdbf8219 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230330155137_supabase_dbdev_80.snap @@ -0,0 +1,306 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230330155137_supabase_dbdev.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'supabase', + 'dbdev', + 'Install pacakges from the dbdev package index', + false, + '{}' + ); + +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_name = 'supabase-dbdev' + ), + (0, 0, 2), + ' + +create schema dbdev; + +create or replace function dbdev.install(package_name text) + returns bool + language plpgsql +as $$ +declare + -- Endpoint + base_url text = ''https://api.database.dev/rest/v1/''; + apikey text = ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyYndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzIsImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJrzM0AQKsu_5k134s''; + + http_ext_schema regnamespace = extnamespace::regnamespace from pg_catalog.pg_extension where extname = ''http'' limit 1; + pgtle_is_available bool = true from pg_catalog.pg_extension where extname = ''pg_tle'' limit 1; + -- HTTP respones + rec jsonb; + status int; + contents json; + + -- Install Record + rec_sql text; + rec_ver text; + rec_from_ver text; + rec_to_ver text; + rec_package_name text; + rec_description text; + rec_requires text[]; +begin + + if http_ext_schema is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the http extension and it is not available''); + end if; + + if pgtle_is_available is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the pgtle extension and it is not available''); + end if; + + ------------------- + -- Base Versions -- + ------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_versions?select=package_name,version,sql,control_description,control_requires&limit=50&package_name=eq.%s'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading versions from dbdev''); + end if; + + if contents is null or json_typeof(contents) <> ''array'' or json_array_length(contents) = 0 then + raise exception using errcode=''22000'', message=format(''No versions for package named named %s'', package_name); + end if; + + for rec_package_name, rec_ver, rec_sql, rec_description, rec_requires in select + (r ->> ''package_name''), + (r ->> ''version''), + (r ->> ''sql''), + (r ->> ''control_description''), + to_json(rec ->> ''control_requires'') #>> ''{}'' + from + json_array_elements(contents) as r + loop + + if not exists ( + select true + from pgtle.available_extension_versions() + where + -- TLE will not allow multiple full install scripts + -- TODO(OR) open upstream issue to discuss + name = rec_package_name + ) then + perform pgtle.install_extension(rec_package_name, rec_ver, rec_package_name, rec_sql); + end if; + end loop; + + ---------------------- + -- Upgrade Versions -- + ---------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_upgrades?select=package_name,from_version,to_version,sql&limit=50&package_name=eq.%s'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading upgrade pathes from dbdev''); + end if; + + if json_typeof(contents) <> ''array'' then + raise exception using errcode=''22000'', message=format(''Invalid response from dbdev upgrade pathes''); + end if; + + for rec_package_name, rec_from_ver, rec_to_ver, rec_sql in select + (r ->> ''package_name''), + (r ->> ''from_version''), + (r ->> ''to_version''), + (r ->> ''sql'') + from + json_array_elements(contents) as r + loop + + if not exists ( + select true + from pgtle.extension_update_paths(rec_package_name) + where + source = rec_from_ver + and target = rec_to_ver + and path is not null + ) then + perform pgtle.install_update_path(rec_package_name, rec_from_ver, rec_to_ver, rec_sql); + end if; + end loop; + + -------------------------- + -- Send Download Notice -- + -------------------------- + -- Notifies dbdev that a package has been downloaded and records IP + user agent so we can compute unique download counts + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''POST'', + format( + ''%srpc/register_download'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header, + (''x-client-info'', ''dbdev/0.0.2'')::http_header + ], + ''application/json'', + json_build_object(''package_name'', $stmt$ || pg_catalog.quote_literal($1) || $stmt$)::text + ) + ) x + limit 1; $stmt$ + into rec; + + return true; +end; +$$; + +', + ' +# dbdev + +dbdev is the SQL client for database.new and is the primary way end users interact with the package (pglet) registry. + +dbdev can be used to load packages from the registry. For example: + +```sql +-- Load the package from the package index +select dbdev.install(''olirice-index_advisor''); +``` +Where `olirice` is the handle of the author and `index_advisor` is the name of the pglet. + +Once installed, pglets are visible in PostgreSQL as extensions. At that point they can be enabled with standard Postgres commands i.e. the `create extension` + +To improve reproducibility, we recommend __always__ specifying the package version in your `create extension` statements. + +For example: +```sql +-- Enable the extension +create extension "olirice-index_advisor" + schema ''public'' + version ''0.1.0''; +``` + +Which creates all tables/indexes/functions/etc specified by the extension. + +## How to Install + +The in-database SQL client for the package registry is named `dbdev`. You can bootstrap the client with: + +```sql +/*--------------------- +---- install dbdev ---- +---------------------- +Requires: + - pg_tle: https://github.com/aws/pg_tle + - pgsql-http: https://github.com/pramsey/pgsql-http +*/ +create extension if not exists http with schema extensions; +create extension if not exists pg_tle; +select pgtle.uninstall_extension_if_exists(''supabase-dbdev''); +drop extension if exists "supabase-dbdev"; +select + pgtle.install_extension( + ''supabase-dbdev'', + resp.contents ->> ''version'', + ''PostgreSQL package manager'', + resp.contents ->> ''sql'' + ) +from http( + ( + ''GET'', + ''https://api.database.dev/rest/v1/'' + || ''package_versions?select=sql,version'' + || ''&package_name=eq.supabase-dbdev'' + || ''&order=version.desc'' + || ''&limit=1'', + array[ + ( + ''apiKey'', + ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJp'' + || ''c3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyY'' + || ''ndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzI'' + || ''sImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJ'' + || ''rzM0AQKsu_5k134s'' + )::http_header + ], + null, + null + ) +) x, +lateral ( + select + ((row_to_json(x) -> ''content'') #>> ''{}'')::json -> 0 +) resp(contents); +create extension "supabase-dbdev"; +select dbdev.install(''supabase-dbdev''); +drop extension if exists "supabase-dbdev"; +create extension "supabase-dbdev"; +``` + +With the client ready, search for packages on [database.dev](database.dev) and install them with + +```sql +select dbdev.install(''handle-package_name''); +create extension "handle-package_name" + schema ''public'' + version ''1.2.3''; +``` +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331145934_burggraf-pg_headerkit_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331145934_burggraf-pg_headerkit_100.snap new file mode 100644 index 000000000..56117f4f5 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331145934_burggraf-pg_headerkit_100.snap @@ -0,0 +1,277 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230331145934_burggraf-pg_headerkit.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'burggraf', + 'pg_headerkit', + 'PostgreSQL functions that read PostgREST headers for adding functionality to your database', + false, + '{}' + ); + +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_name = 'burggraf-pg_headerkit' + ), + (1, 0, 0), + ' +CREATE SCHEMA IF NOT EXISTS hdr; + +CREATE TABLE IF NOT EXISTS hdr.allow_list ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + ip inet NOT NULL, + created_at timestamp with time zone NOT NULL DEFAULT now(), + updated_at timestamp with time zone NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS hdr.deny_list ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + ip inet NOT NULL, + created_at timestamp with time zone NOT NULL DEFAULT now(), + updated_at timestamp with time zone NOT NULL DEFAULT now() +); + +-- get all header values as a json object +CREATE OR REPLACE FUNCTION hdr.headers() RETURNS json + LANGUAGE sql STABLE + AS $$ + SELECT COALESCE(current_setting(''request.headers'', true)::json, ''{}''::json); +$$; + +-- get a header value +CREATE OR REPLACE FUNCTION hdr.header(item text) RETURNS text + LANGUAGE sql STABLE + AS $$ + SELECT COALESCE((current_setting(''request.headers'', true)::json)->>item, '''') +$$; + +-- get the ip address of the current user +CREATE OR REPLACE FUNCTION hdr.ip() RETURNS text + LANGUAGE sql STABLE + AS $$ + SELECT SPLIT_PART(hdr.header(''x-forwarded-for'') || '','', '','', 1) +$$; + +-- get the allow list +CREATE OR REPLACE FUNCTION hdr.allow_list() RETURNS inet[] + LANGUAGE sql IMMUTABLE + AS $$ + SELECT array_agg(ip) FROM (SELECT ip FROM hdr.allow_list) AS ip; +$$; + +-- get the deny list +CREATE OR REPLACE FUNCTION hdr.deny_list() RETURNS inet[] + LANGUAGE sql IMMUTABLE + AS $$ + SELECT array_agg(ip) FROM (SELECT ip FROM hdr.deny_list) AS ip; +$$; + +-- Is the given ip in the deny list? +CREATE OR REPLACE FUNCTION hdr.in_deny_list(ip inet) RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT + ip = ANY (hdr.deny_list()) +$$; + +-- Is the current user''s ip in the deny list? +CREATE OR REPLACE FUNCTION hdr.in_deny_list() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT CASE + WHEN hdr.ip() = '''' THEN false + ELSE + (hdr.ip())::inet = ANY (hdr.deny_list()) + END +$$; + +-- Is the given ip in the allow list? +CREATE OR REPLACE FUNCTION hdr.in_allow_list(ip inet) RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT + ip = ANY (hdr.allow_list()) +$$; + +-- Is the current user''s ip in the allow list? +CREATE OR REPLACE FUNCTION hdr.in_allow_list() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT CASE + WHEN hdr.ip() = '''' THEN false + ELSE + (hdr.ip())::inet = ANY (hdr.allow_list()) + END +$$; + +-- get host, i.e. "localhost:3000" +CREATE OR REPLACE FUNCTION hdr.host() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''host'') +$$; + +-- get origin, i.e. "http://localhost:8100" +CREATE OR REPLACE FUNCTION hdr.origin() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''origin'') +$$; + +-- get referer, i.e. "http://localhost:8100/" +CREATE OR REPLACE FUNCTION hdr.referer() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''referer'') +$$; + +-- get user-agent string +CREATE OR REPLACE FUNCTION hdr.agent() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''user-agent'') +$$; + +-- get x-client-info, i.e. "supabase-js/1.35.7" +CREATE OR REPLACE FUNCTION hdr.client() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''x-client-info'') +$$; + +-- get role (consumer), i.e. "anon-key" +CREATE OR REPLACE FUNCTION hdr.role() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''x-consumer-username'') +$$; + +-- get consumer, i.e. "anon-key" +CREATE OR REPLACE FUNCTION hdr.consumer() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''x-consumer-username'') +$$; + +-- get api server, i.e. "xxxxxxxxxxxxxxxx.supabase.co" +CREATE OR REPLACE FUNCTION hdr.api_host() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''x-forwarded-host'') +$$; + +-- get api server domain, i.e. "xxxxxxxxxxxxxxxx.supabase.co" +CREATE OR REPLACE FUNCTION hdr.domain() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''x-forwarded-host'') +$$; + +-- get project ref #, i.e. "xxxxxxxxxxxxxxxx" +CREATE OR REPLACE FUNCTION hdr.projectref() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''x-forwarded-host'') + SELECT SPLIT_PART(hdr.header(''x-forwarded-host'') || ''.'', ''.'', 1) +$$; + +-- get project ref #, i.e. "xxxxxxxxxxxxxxxx" +CREATE OR REPLACE FUNCTION hdr.ref() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''x-forwarded-host'') + SELECT SPLIT_PART(hdr.header(''x-forwarded-host'') || ''.'', ''.'', 1) +$$; + +-- ********************************************** +-- ********* user-agent parse functions ********* +-- ********************************************** + +-- user-agent parsing for mobile +CREATE OR REPLACE FUNCTION hdr.is_mobile() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''user-agent'') ILIKE ''%mobile%'' +$$; + +-- user-agent parsing for iPhone +CREATE OR REPLACE FUNCTION hdr.is_iphone() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''user-agent'') ILIKE ''%iphone%'' +$$; + +-- user-agent parsing for iPad +CREATE OR REPLACE FUNCTION hdr.is_ipad() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''user-agent'') ILIKE ''%ipad%'' +$$; + +-- user-agent parsing for Android +CREATE OR REPLACE FUNCTION hdr.is_android() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''user-agent'') ILIKE ''%android%'' +$$; + +', + ' +# pg_headerkit: PostgREST Header Kit +A set of functions for adding special features to your application that use PostgREST API calls to your PostgreSQL database. These functions can be used inside PostgreSQL functions that can give your application the following capabilities at the database level: + +- [x] rate limiting +- [x] IP allowlisting +- [x] IP denylisting +- [x] request logging +- [x] request filtering +- [x] request routing +- [x] user allowlisting by uid or email (Supabase-specific) +- [x] user denylisting by uid or email (Supabase-specific) + +### Article +See: [PostgREST Header Hacking](https://github.com/burggraf/postgrest-header-hacking) + +### Function Reference + +| function | description | parameters | returns | +| ------------------------------ | ------------------------------------------------------- | ----------- | ------------------------------ | +| hdr.headers() | get all header values as a json object | none | json object | +| hdr.header(item text) | get a header value | item (text) | text | +| hdr.ip() | get the ip address of the current user | none | text | +| hdr.allow_list() | get the allow list of ip addresses | none | inet[] (array of ip addresses) | +| hdr.deny_list() | get the deny list of ip addresses | none | inet[] (array of ip addresses) | +| hdr.in_deny_list(ip inet) | determine if the given ip is in the deny list | ip (inet) | boolean | +| hdr.in_allow_list(ip inet) | determine if the given ip is in the allow list | ip (inet) | boolean | +| hdr.in_deny_list() | determine if the current user''s ip is in the deny list | none | boolean | +| hdr.in_allow_list() | determine if the current user''s ip is in the allow list | none | boolean | +| hdr.host() | get host, i.e. "localhost:3000" | none | text | +| hdr.origin() | get origin, i.e. "http://localhost:8100" | none | text | +| hdr.referer() | get referer, i.e. "http://localhost:8100/" | none | text | +| hdr.agent() | get user-agent string | none | text | +| hdr.client() | get x-client-info, i.e. "supabase-js/1.35.7" | none | text | +| hdr.role()
hdr.consumer() | get role (consumer), i.e. "anon-key" | none | text | +| hdr.api_host()
hdr.domain() | get api server, i.e. "xxxxxxxxxxxxxxxx.supabase.co" | none | text | +| hdr.projectref()
hdr.ref() | get project ref #, i.e. "xxxxxxxxxxxxxxxx" | none | text | +| hdr.is_mobile() | is mobile? | none | boolean | +| hdr.is_iphone() | is iphone? | none | boolean | +| hdr.is_ipad() | is ipad? | none | boolean | +| hdr.is_android() | is android? | none | boolean | +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331145934_burggraf-pg_headerkit_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331145934_burggraf-pg_headerkit_80.snap new file mode 100644 index 000000000..519b0d147 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331145934_burggraf-pg_headerkit_80.snap @@ -0,0 +1,282 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230331145934_burggraf-pg_headerkit.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'burggraf', + 'pg_headerkit', + 'PostgreSQL functions that read PostgREST headers for adding functionality to your database', + false, + '{}' + ); + +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_name = 'burggraf-pg_headerkit' + ), + (1, 0, 0), + ' +CREATE SCHEMA IF NOT EXISTS hdr; + +CREATE TABLE IF NOT EXISTS hdr.allow_list ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + ip inet NOT NULL, + created_at timestamp with time zone NOT NULL DEFAULT now(), + updated_at timestamp with time zone NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS hdr.deny_list ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + ip inet NOT NULL, + created_at timestamp with time zone NOT NULL DEFAULT now(), + updated_at timestamp with time zone NOT NULL DEFAULT now() +); + +-- get all header values as a json object +CREATE OR REPLACE FUNCTION hdr.headers() RETURNS json + LANGUAGE sql STABLE + AS $$ + SELECT COALESCE(current_setting(''request.headers'', true)::json, ''{}''::json); +$$; + +-- get a header value +CREATE OR REPLACE FUNCTION hdr.header(item text) RETURNS text + LANGUAGE sql STABLE + AS $$ + SELECT COALESCE((current_setting(''request.headers'', true)::json)->>item, '''') +$$; + +-- get the ip address of the current user +CREATE OR REPLACE FUNCTION hdr.ip() RETURNS text + LANGUAGE sql STABLE + AS $$ + SELECT SPLIT_PART(hdr.header(''x-forwarded-for'') || '','', '','', 1) +$$; + +-- get the allow list +CREATE OR REPLACE FUNCTION hdr.allow_list() RETURNS inet[] + LANGUAGE sql IMMUTABLE + AS $$ + SELECT array_agg(ip) FROM (SELECT ip FROM hdr.allow_list) AS ip; +$$; + +-- get the deny list +CREATE OR REPLACE FUNCTION hdr.deny_list() RETURNS inet[] + LANGUAGE sql IMMUTABLE + AS $$ + SELECT array_agg(ip) FROM (SELECT ip FROM hdr.deny_list) AS ip; +$$; + +-- Is the given ip in the deny list? +CREATE OR REPLACE FUNCTION hdr.in_deny_list(ip inet) RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT + ip = ANY (hdr.deny_list()) +$$; + +-- Is the current user''s ip in the deny list? +CREATE OR REPLACE FUNCTION hdr.in_deny_list() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT CASE + WHEN hdr.ip() = '''' THEN false + ELSE + (hdr.ip())::inet = ANY (hdr.deny_list()) + END +$$; + +-- Is the given ip in the allow list? +CREATE OR REPLACE FUNCTION hdr.in_allow_list(ip inet) RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT + ip = ANY (hdr.allow_list()) +$$; + +-- Is the current user''s ip in the allow list? +CREATE OR REPLACE FUNCTION hdr.in_allow_list() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT CASE + WHEN hdr.ip() = '''' THEN false + ELSE + (hdr.ip())::inet = ANY (hdr.allow_list()) + END +$$; + +-- get host, i.e. "localhost:3000" +CREATE OR REPLACE FUNCTION hdr.host() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''host'') +$$; + +-- get origin, i.e. "http://localhost:8100" +CREATE OR REPLACE FUNCTION hdr.origin() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''origin'') +$$; + +-- get referer, i.e. "http://localhost:8100/" +CREATE OR REPLACE FUNCTION hdr.referer() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''referer'') +$$; + +-- get user-agent string +CREATE OR REPLACE FUNCTION hdr.agent() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''user-agent'') +$$; + +-- get x-client-info, i.e. "supabase-js/1.35.7" +CREATE OR REPLACE FUNCTION hdr.client() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''x-client-info'') +$$; + +-- get role (consumer), i.e. "anon-key" +CREATE OR REPLACE FUNCTION hdr.role() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''x-consumer-username'') +$$; + +-- get consumer, i.e. "anon-key" +CREATE OR REPLACE FUNCTION hdr.consumer() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''x-consumer-username'') +$$; + +-- get api server, i.e. "xxxxxxxxxxxxxxxx.supabase.co" +CREATE OR REPLACE FUNCTION hdr.api_host() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''x-forwarded-host'') +$$; + +-- get api server domain, i.e. "xxxxxxxxxxxxxxxx.supabase.co" +CREATE OR REPLACE FUNCTION hdr.domain() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''x-forwarded-host'') +$$; + +-- get project ref #, i.e. "xxxxxxxxxxxxxxxx" +CREATE OR REPLACE FUNCTION hdr.projectref() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''x-forwarded-host'') + SELECT SPLIT_PART(hdr.header(''x-forwarded-host'') || ''.'', ''.'', 1) +$$; + +-- get project ref #, i.e. "xxxxxxxxxxxxxxxx" +CREATE OR REPLACE FUNCTION hdr.ref() RETURNS text + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''x-forwarded-host'') + SELECT SPLIT_PART(hdr.header(''x-forwarded-host'') || ''.'', ''.'', 1) +$$; + +-- ********************************************** +-- ********* user-agent parse functions ********* +-- ********************************************** + +-- user-agent parsing for mobile +CREATE OR REPLACE FUNCTION hdr.is_mobile() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''user-agent'') ILIKE ''%mobile%'' +$$; + +-- user-agent parsing for iPhone +CREATE OR REPLACE FUNCTION hdr.is_iphone() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''user-agent'') ILIKE ''%iphone%'' +$$; + +-- user-agent parsing for iPad +CREATE OR REPLACE FUNCTION hdr.is_ipad() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''user-agent'') ILIKE ''%ipad%'' +$$; + +-- user-agent parsing for Android +CREATE OR REPLACE FUNCTION hdr.is_android() RETURNS boolean + LANGUAGE sql IMMUTABLE + AS $$ + SELECT hdr.header(''user-agent'') ILIKE ''%android%'' +$$; + +', + ' +# pg_headerkit: PostgREST Header Kit +A set of functions for adding special features to your application that use PostgREST API calls to your PostgreSQL database. These functions can be used inside PostgreSQL functions that can give your application the following capabilities at the database level: + +- [x] rate limiting +- [x] IP allowlisting +- [x] IP denylisting +- [x] request logging +- [x] request filtering +- [x] request routing +- [x] user allowlisting by uid or email (Supabase-specific) +- [x] user denylisting by uid or email (Supabase-specific) + +### Article +See: [PostgREST Header Hacking](https://github.com/burggraf/postgrest-header-hacking) + +### Function Reference + +| function | description | parameters | returns | +| ------------------------------ | ------------------------------------------------------- | ----------- | ------------------------------ | +| hdr.headers() | get all header values as a json object | none | json object | +| hdr.header(item text) | get a header value | item (text) | text | +| hdr.ip() | get the ip address of the current user | none | text | +| hdr.allow_list() | get the allow list of ip addresses | none | inet[] (array of ip addresses) | +| hdr.deny_list() | get the deny list of ip addresses | none | inet[] (array of ip addresses) | +| hdr.in_deny_list(ip inet) | determine if the given ip is in the deny list | ip (inet) | boolean | +| hdr.in_allow_list(ip inet) | determine if the given ip is in the allow list | ip (inet) | boolean | +| hdr.in_deny_list() | determine if the current user''s ip is in the deny list | none | boolean | +| hdr.in_allow_list() | determine if the current user''s ip is in the allow list | none | boolean | +| hdr.host() | get host, i.e. "localhost:3000" | none | text | +| hdr.origin() | get origin, i.e. "http://localhost:8100" | none | text | +| hdr.referer() | get referer, i.e. "http://localhost:8100/" | none | text | +| hdr.agent() | get user-agent string | none | text | +| hdr.client() | get x-client-info, i.e. "supabase-js/1.35.7" | none | text | +| hdr.role()
hdr.consumer() | get role (consumer), i.e. "anon-key" | none | text | +| hdr.api_host()
hdr.domain() | get api server, i.e. "xxxxxxxxxxxxxxxx.supabase.co" | none | text | +| hdr.projectref()
hdr.ref() | get project ref #, i.e. "xxxxxxxxxxxxxxxx" | none | text | +| hdr.is_mobile() | is mobile? | none | boolean | +| hdr.is_iphone() | is iphone? | none | boolean | +| hdr.is_ipad() | is ipad? | none | boolean | +| hdr.is_android() | is android? | none | boolean | +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163908_olirice-index_advisor_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163908_olirice-index_advisor_100.snap new file mode 100644 index 000000000..da68b86e8 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163908_olirice-index_advisor_100.snap @@ -0,0 +1,334 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230331163908_olirice-index_advisor.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'olirice', + 'index_advisor', + 'Recommend indexes for a given SQL query', + true, + '{hypopg}' + ); + +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_name = 'olirice-index_advisor' + ), + (0, 1, 0), + ' + + +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + hypopg_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''hypopg'' + and installed_version is not null + ); + begin + + if not hypopg_exists then + raise + exception ''"olirice-index_advisor" requires "hypopg"'' + using hint = ''Run "create extension hypopg" and try again''; + end if; + end +$$; + + +create type index_advisor_output as ( + index_statements text[], + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb +); + +create function index_advisor( + query text +) + returns table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[] + ) + volatile + language plpgsql + as $$ +declare + n_args int; + prepared_statement_name text = ''index_advisor_working_statement''; + hypopg_schema_name text = (select extnamespace::regnamespace::text from pg_extension where extname = ''hypopg''); + explain_plan_statement text; + rec record; + plan_initial jsonb; + plan_final jsonb; + statements text[] = ''{}''; +begin + + -- Disallow multiple statements + if query ilike ''%;%'' then + raise exception ''query must not contain a semicolon''; + end if; + + -- Hack to support PostgREST because the prepared statement for args incorrectly defaults to text + query := replace(query, ''WITH pgrst_payload AS (SELECT $1 AS json_data)'', ''WITH pgrst_payload AS (SELECT $1::json AS json_data)''); + + -- Create a prepared statement for the given query + deallocate all; + execute format(''prepare %I as %s'', prepared_statement_name, query); + + -- Detect how many arguments are present in the prepared statement + n_args = ( + select + coalesce(array_length(parameter_types, 1), 0) + from + pg_prepared_statements + where + name = prepared_statement_name + limit + 1 + ); + + -- Create a SQL statement that can be executed to collect the explain plan + explain_plan_statement = format( + ''set local plan_cache_mode = force_generic_plan; explain (format json) execute %I%s'', + --''explain (format json) execute %I%s'', + prepared_statement_name, + case + when n_args = 0 then '''' + else format( + ''(%s)'', array_to_string(array_fill(''null''::text, array[n_args]), '','') + ) + end + ); + + -- Store the query plan before any new indexes + execute explain_plan_statement into plan_initial; + + -- Create possible indexes + for rec in ( + with extension_regclass as ( + select + distinct objid as oid + from + pg_depend + where + deptype = ''e'' + ) + select + pc.relnamespace::regnamespace::text as schema_name, + pc.relname as table_name, + pa.attname as column_name, + format( + ''select %I.hypopg_create_index($i$create index on %I.%I(%I)$i$)'', + hypopg_schema_name, + pc.relnamespace::regnamespace::text, + pc.relname, + pa.attname + ) hypopg_statement + from + pg_catalog.pg_class pc + join pg_catalog.pg_attribute pa + on pc.oid = pa.attrelid + left join extension_regclass er + on pc.oid = er.oid + left join pg_index pi + on pc.oid = pi.indrelid + and (select array_agg(x) from unnest(pi.indkey) v(x)) = array[pa.attnum] + and pi.indexprs is null -- ignore expression indexes + and pi.indpred is null -- ignore partial indexes + where + pc.relnamespace::regnamespace::text not in ( -- ignore schema list + ''pg_catalog'', ''pg_toast'', ''information_schema'' + ) + and er.oid is null -- ignore entities owned by extensions + and pc.relkind in (''r'', ''m'') -- regular tables, and materialized views + and pc.relpersistence = ''p'' -- permanent tables (not unlogged or temporary) + and pa.attnum > 0 + and not pa.attisdropped + and pi.indrelid is null + and pa.atttypid in (20,16,1082,1184,1114,701,23,21,700,1083,2950,1700,25,18,1042,1043) + ) + loop + -- Create the hypothetical index + execute rec.hypopg_statement; + end loop; + + -- Create a prepared statement for the given query + -- The original prepared statement MUST be dropped because its plan is cached + execute format(''deallocate %I'', prepared_statement_name); + execute format(''prepare %I as %s'', prepared_statement_name, query); + + -- Store the query plan after new indexes + execute explain_plan_statement into plan_final; + + + -- Idenfity referenced indexes in new plan + execute format( + ''select + coalesce(array_agg(hypopg_get_indexdef(indexrelid) order by indrelid, indkey::text), $i${}$i$::text[]) + from + %I.hypopg() + where + %s ilike ($i$%%$i$ || indexname || $i$%%$i$) + '', + hypopg_schema_name, + quote_literal(plan_final)::text + ) into statements; + + -- Reset all hypothetical indexes + perform hypopg_reset(); + + -- Reset prepared statements + deallocate all; + + return query values ( + (plan_initial -> 0 -> ''Plan'' -> ''Startup Cost''), + (plan_final -> 0 -> ''Plan'' -> ''Startup Cost''), + (plan_initial -> 0 -> ''Plan'' -> ''Total Cost''), + (plan_final -> 0 -> ''Plan'' -> ''Total Cost''), + statements::text[] + ); + +end; +$$; + +', + ' + +# index_advisor + +`index_advisor` is an extension that recommends indexes to improve performance of a given query. + +## Installation + +Note: + +`hypopg` is a dependency of index_advisor. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +```sql +select dbdev.install(''olirice-index_advisor''); +create extension if not exists hypopg; +create extension "olirice-index_advisor" cascade; +``` + +## Example + +For a simple example, consider the following table: + +```sql +create table book( + id int primary key, + title text not null +); +``` + +Lets say we want to query `book` by `title`, and return the relevant `id`. +That query would be `select book.id from book where title = $1`. + +We can get `index_advisor` to recommend indexes that would improve performance on that query as follows: + + +```sql + +select + * +from + index_advisor(''select book.id from book where title = $1''); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements +---------------------+--------------------+-------------------+------------------+----------------------------------------------------- + 0.00 | 1.17 | 25.88 | 6.40 | {"CREATE INDEX ON public.book USING btree (title)"}, +(1 row) +``` + +where the output columns show top level statistics from the query explain plan (startup_cost, total_cost) and an array of `index_statements` that improve `total_cost`. + +## Features: + +- Generic parameters e.g. `$1`, `$2` +- Support for Materialized Views +- Identifies Tables/Columns Oobfuscaed by Views + +## Usage + +`index_advisor` is not limited to simple use cases. A more complex example could be: + +```sql +select + * +from + index_advisor('' + select + book.id, + book.title, + publisher.name as publisher_name, + author.name as author_name, + review.body review_body + from + book + join publisher + on book.publisher_id = publisher.id + join author + on book.author_id = author.id + join review + on book.id = review.book_id + where + author.id = $1 + and publisher.id = $2 + ''); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements +---------------------+--------------------+-------------------+------------------+---------------------------------------------------------- + 27.26 | 12.77 | 68.48 | 42.37 | {"CREATE INDEX ON public.book USING btree (author_id)", + "CREATE INDEX ON public.book USING btree (publisher_id)", + "CREATE INDEX ON public.review USING btree (book_id)"} +(1 row) +``` + +Note: the referenced tables must exist. + +## API + +```sql +index_advisor(query text) +returns + table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[] + ) +``` + +#### Description +For a given *query*, searches for a set of SQL DDL `create index` statements that improve the query''s execution time; + +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163908_olirice-index_advisor_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163908_olirice-index_advisor_80.snap new file mode 100644 index 000000000..fb253ab4f --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163908_olirice-index_advisor_80.snap @@ -0,0 +1,339 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230331163908_olirice-index_advisor.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'olirice', + 'index_advisor', + 'Recommend indexes for a given SQL query', + true, + '{hypopg}' + ); + +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_name = 'olirice-index_advisor' + ), + (0, 1, 0), + ' + + +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + hypopg_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''hypopg'' + and installed_version is not null + ); + begin + + if not hypopg_exists then + raise + exception ''"olirice-index_advisor" requires "hypopg"'' + using hint = ''Run "create extension hypopg" and try again''; + end if; + end +$$; + + +create type index_advisor_output as ( + index_statements text[], + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb +); + +create function index_advisor( + query text +) + returns table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[] + ) + volatile + language plpgsql + as $$ +declare + n_args int; + prepared_statement_name text = ''index_advisor_working_statement''; + hypopg_schema_name text = (select extnamespace::regnamespace::text from pg_extension where extname = ''hypopg''); + explain_plan_statement text; + rec record; + plan_initial jsonb; + plan_final jsonb; + statements text[] = ''{}''; +begin + + -- Disallow multiple statements + if query ilike ''%;%'' then + raise exception ''query must not contain a semicolon''; + end if; + + -- Hack to support PostgREST because the prepared statement for args incorrectly defaults to text + query := replace(query, ''WITH pgrst_payload AS (SELECT $1 AS json_data)'', ''WITH pgrst_payload AS (SELECT $1::json AS json_data)''); + + -- Create a prepared statement for the given query + deallocate all; + execute format(''prepare %I as %s'', prepared_statement_name, query); + + -- Detect how many arguments are present in the prepared statement + n_args = ( + select + coalesce(array_length(parameter_types, 1), 0) + from + pg_prepared_statements + where + name = prepared_statement_name + limit + 1 + ); + + -- Create a SQL statement that can be executed to collect the explain plan + explain_plan_statement = format( + ''set local plan_cache_mode = force_generic_plan; explain (format json) execute %I%s'', + --''explain (format json) execute %I%s'', + prepared_statement_name, + case + when n_args = 0 then '''' + else format( + ''(%s)'', array_to_string(array_fill(''null''::text, array[n_args]), '','') + ) + end + ); + + -- Store the query plan before any new indexes + execute explain_plan_statement into plan_initial; + + -- Create possible indexes + for rec in ( + with extension_regclass as ( + select + distinct objid as oid + from + pg_depend + where + deptype = ''e'' + ) + select + pc.relnamespace::regnamespace::text as schema_name, + pc.relname as table_name, + pa.attname as column_name, + format( + ''select %I.hypopg_create_index($i$create index on %I.%I(%I)$i$)'', + hypopg_schema_name, + pc.relnamespace::regnamespace::text, + pc.relname, + pa.attname + ) hypopg_statement + from + pg_catalog.pg_class pc + join pg_catalog.pg_attribute pa + on pc.oid = pa.attrelid + left join extension_regclass er + on pc.oid = er.oid + left join pg_index pi + on pc.oid = pi.indrelid + and (select array_agg(x) from unnest(pi.indkey) v(x)) = array[pa.attnum] + and pi.indexprs is null -- ignore expression indexes + and pi.indpred is null -- ignore partial indexes + where + pc.relnamespace::regnamespace::text not in ( -- ignore schema list + ''pg_catalog'', ''pg_toast'', ''information_schema'' + ) + and er.oid is null -- ignore entities owned by extensions + and pc.relkind in (''r'', ''m'') -- regular tables, and materialized views + and pc.relpersistence = ''p'' -- permanent tables (not unlogged or temporary) + and pa.attnum > 0 + and not pa.attisdropped + and pi.indrelid is null + and pa.atttypid in (20,16,1082,1184,1114,701,23,21,700,1083,2950,1700,25,18,1042,1043) + ) + loop + -- Create the hypothetical index + execute rec.hypopg_statement; + end loop; + + -- Create a prepared statement for the given query + -- The original prepared statement MUST be dropped because its plan is cached + execute format(''deallocate %I'', prepared_statement_name); + execute format(''prepare %I as %s'', prepared_statement_name, query); + + -- Store the query plan after new indexes + execute explain_plan_statement into plan_final; + + + -- Idenfity referenced indexes in new plan + execute format( + ''select + coalesce(array_agg(hypopg_get_indexdef(indexrelid) order by indrelid, indkey::text), $i${}$i$::text[]) + from + %I.hypopg() + where + %s ilike ($i$%%$i$ || indexname || $i$%%$i$) + '', + hypopg_schema_name, + quote_literal(plan_final)::text + ) into statements; + + -- Reset all hypothetical indexes + perform hypopg_reset(); + + -- Reset prepared statements + deallocate all; + + return query values ( + (plan_initial -> 0 -> ''Plan'' -> ''Startup Cost''), + (plan_final -> 0 -> ''Plan'' -> ''Startup Cost''), + (plan_initial -> 0 -> ''Plan'' -> ''Total Cost''), + (plan_final -> 0 -> ''Plan'' -> ''Total Cost''), + statements::text[] + ); + +end; +$$; + +', + ' + +# index_advisor + +`index_advisor` is an extension that recommends indexes to improve performance of a given query. + +## Installation + +Note: + +`hypopg` is a dependency of index_advisor. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +```sql +select dbdev.install(''olirice-index_advisor''); +create extension if not exists hypopg; +create extension "olirice-index_advisor" cascade; +``` + +## Example + +For a simple example, consider the following table: + +```sql +create table book( + id int primary key, + title text not null +); +``` + +Lets say we want to query `book` by `title`, and return the relevant `id`. +That query would be `select book.id from book where title = $1`. + +We can get `index_advisor` to recommend indexes that would improve performance on that query as follows: + + +```sql + +select + * +from + index_advisor(''select book.id from book where title = $1''); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements +---------------------+--------------------+-------------------+------------------+----------------------------------------------------- + 0.00 | 1.17 | 25.88 | 6.40 | {"CREATE INDEX ON public.book USING btree (title)"}, +(1 row) +``` + +where the output columns show top level statistics from the query explain plan (startup_cost, total_cost) and an array of `index_statements` that improve `total_cost`. + +## Features: + +- Generic parameters e.g. `$1`, `$2` +- Support for Materialized Views +- Identifies Tables/Columns Oobfuscaed by Views + +## Usage + +`index_advisor` is not limited to simple use cases. A more complex example could be: + +```sql +select + * +from + index_advisor('' + select + book.id, + book.title, + publisher.name as publisher_name, + author.name as author_name, + review.body review_body + from + book + join publisher + on book.publisher_id = publisher.id + join author + on book.author_id = author.id + join review + on book.id = review.book_id + where + author.id = $1 + and publisher.id = $2 + ''); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements +---------------------+--------------------+-------------------+------------------+---------------------------------------------------------- + 27.26 | 12.77 | 68.48 | 42.37 | {"CREATE INDEX ON public.book USING btree (author_id)", + "CREATE INDEX ON public.book USING btree (publisher_id)", + "CREATE INDEX ON public.review USING btree (book_id)"} +(1 row) +``` + +Note: the referenced tables must exist. + +## API + +```sql +index_advisor(query text) +returns + table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[] + ) +``` + +#### Description +For a given *query*, searches for a set of SQL DDL `create index` statements that improve the query''s execution time; + +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163909_olirice-read_once_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163909_olirice-read_once_100.snap new file mode 100644 index 000000000..c7b46dd26 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163909_olirice-read_once_100.snap @@ -0,0 +1,172 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230331163909_olirice-read_once.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'olirice', + 'read_once', + 'Send messages that can only be read once', + false, + '{pg_cron}' + ); + +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_name = 'olirice-read_once' + ), + (0, 3, 1), + ' + +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + pg_cron_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''pg_cron'' + and installed_version is not null + ); + begin + + if not pg_cron_exists then + raise + exception ''"olirice-read_once" requires "pg_cron"'' + using hint = ''Run "create extension pg_cron" and try again''; + end if; + end +$$; + + +create schema read_once; + +create unlogged table read_once.messages( + id uuid primary key default gen_random_uuid(), + contents text not null default '''', + created_at timestamp default now() +); + +revoke all on read_once.messages from public; +revoke usage on schema read_once from public; + +create or replace function send_message( + contents text +) + returns uuid + security definer + volatile + strict + language sql + as +$$ + insert into read_once.messages(contents) + values ($1) + returning id; +$$; + +create or replace function read_message(id uuid) + returns text + security definer + volatile + strict + language sql + as +$$ + delete from read_once.messages + where read_once.messages.id = $1 + returning contents; +$$ +', + ' + +# read_once + +A Supabase application for sending messages that can only be read once + +Features: +- messages can only be read one time +- messages are not logged in PostgreSQL write-ahead-log (WAL) + +## Installation + +`pg_cron` is a dependency of `read_once`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + +To expose the `send_message` and `read_message` functions over HTTP, install the extension in a schema that is on the search_path. + + +For example: +```sql +select dbdev.install(''olirice-read_once''); +create extension if not exists pg_cron; +create extension "olirice-read_once" + schema public + version ''0.3.1''; +``` + + +## HTTP API + +### Create a Message + +```sh +curl -X POST https://.supabase.co/rest/v1/rpc/send_message \ + -H ''apiKey: '' \ + -H ''Content-Type: application/json'' + --data-raw ''{"contents": "hello, dbdev!"} + +# Returns: "2989156b-2356-4543-9d1b-19dfb8ec3268" +``` + +### Read a Message + +```sh +curl -X https://.supabase.co/rest/v1/rpc/read_message + -H ''apiKey: '' \ + -H ''Content-Type: application/json'' \ + --data-raw ''{"id": "2989156b-2356-4543-9d1b-19dfb8ec3268"} + +# Returns: "hello, dbdev!" +``` + + +## SQL API + +### Create a Message + +```sql +-- Creates a new messages and returns its unique id +create or replace function send_message( + contents text +) + returns uuid +``` + +### Read a Message + +```sql +-- Read a message by its id +create or replace function read_message( + id uuid +) + returns text +``` +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163909_olirice-read_once_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163909_olirice-read_once_80.snap new file mode 100644 index 000000000..002c8e806 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230331163909_olirice-read_once_80.snap @@ -0,0 +1,177 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230331163909_olirice-read_once.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'olirice', + 'read_once', + 'Send messages that can only be read once', + false, + '{pg_cron}' + ); + +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_name = 'olirice-read_once' + ), + (0, 3, 1), + ' + +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + pg_cron_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''pg_cron'' + and installed_version is not null + ); + begin + + if not pg_cron_exists then + raise + exception ''"olirice-read_once" requires "pg_cron"'' + using hint = ''Run "create extension pg_cron" and try again''; + end if; + end +$$; + + +create schema read_once; + +create unlogged table read_once.messages( + id uuid primary key default gen_random_uuid(), + contents text not null default '''', + created_at timestamp default now() +); + +revoke all on read_once.messages from public; +revoke usage on schema read_once from public; + +create or replace function send_message( + contents text +) + returns uuid + security definer + volatile + strict + language sql + as +$$ + insert into read_once.messages(contents) + values ($1) + returning id; +$$; + +create or replace function read_message(id uuid) + returns text + security definer + volatile + strict + language sql + as +$$ + delete from read_once.messages + where read_once.messages.id = $1 + returning contents; +$$ +', + ' + +# read_once + +A Supabase application for sending messages that can only be read once + +Features: +- messages can only be read one time +- messages are not logged in PostgreSQL write-ahead-log (WAL) + +## Installation + +`pg_cron` is a dependency of `read_once`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + +To expose the `send_message` and `read_message` functions over HTTP, install the extension in a schema that is on the search_path. + + +For example: +```sql +select dbdev.install(''olirice-read_once''); +create extension if not exists pg_cron; +create extension "olirice-read_once" + schema public + version ''0.3.1''; +``` + + +## HTTP API + +### Create a Message + +```sh +curl -X POST https://.supabase.co/rest/v1/rpc/send_message \ + -H ''apiKey: '' \ + -H ''Content-Type: application/json'' + --data-raw ''{"contents": "hello, dbdev!"} + +# Returns: "2989156b-2356-4543-9d1b-19dfb8ec3268" +``` + +### Read a Message + +```sh +curl -X https://.supabase.co/rest/v1/rpc/read_message + -H ''apiKey: '' \ + -H ''Content-Type: application/json'' \ + --data-raw ''{"id": "2989156b-2356-4543-9d1b-19dfb8ec3268"} + +# Returns: "hello, dbdev!" +``` + + +## SQL API + +### Create a Message + +```sql +-- Creates a new messages and returns its unique id +create or replace function send_message( + contents text +) + returns uuid +``` + +### Read a Message + +```sql +-- Read a message by its id +create or replace function read_message( + id uuid +) + returns text +``` +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230404162614_michelp-adminpack_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230404162614_michelp-adminpack_100.snap new file mode 100644 index 000000000..1f5b37208 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230404162614_michelp-adminpack_100.snap @@ -0,0 +1,1582 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230404162614_michelp-adminpack.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'michelp', + 'adminpack', + 'A bunch of useful queries for DBA to manage and inspect databases', + false, + '{}' + ); + +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_name = 'michelp-adminpack' + ), + (0, 0, 1), + ' +-- From: https://github.com/ioguix/pgsql-bloat-estimation + +-- Copyright (c) 2015-2019, Jehan-Guillaume (ioguix) de Rorthais +-- All rights reserved. + +-- Redistribution and use in source and binary forms, with or without +-- modification, are permitted provided that the following conditions are met: + +-- * Redistributions of source code must retain the above copyright notice, this +-- list of conditions and the following disclaimer. + +-- * Redistributions in binary form must reproduce the above copyright notice, +-- this list of conditions and the following disclaimer in the documentation +-- and/or other materials provided with the distribution. + +-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +-- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +-- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +-- DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +-- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +-- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +-- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +-- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +-- OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +-- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-- WARNING: executed with a non-superuser role, the query inspect only index on tables you are granted to read. +-- WARNING: rows with is_na = ''t'' are known to have bad statistics ("name" type is not supported). +-- This query is compatible with PostgreSQL 8.2 and after +CREATE VIEW index_bloat AS SELECT current_database(), nspname AS schemaname, tblname, idxname, bs*(relpages)::bigint AS real_size, + bs*(relpages-est_pages)::bigint AS extra_size, + 100 * (relpages-est_pages)::float / relpages AS extra_pct, + fillfactor, + CASE WHEN relpages > est_pages_ff + THEN bs*(relpages-est_pages_ff) + ELSE 0 + END AS bloat_size, + 100 * (relpages-est_pages_ff)::float / relpages AS bloat_pct, + is_na + -- , 100-(pst).avg_leaf_density AS pst_avg_bloat, est_pages, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples, relpages -- (DEBUG INFO) +FROM ( + SELECT coalesce(1 + + ceil(reltuples/floor((bs-pageopqdata-pagehdr)/(4+nulldatahdrwidth)::float)), 0 -- ItemIdData size + computed avg size of a tuple (nulldatahdrwidth) + ) AS est_pages, + coalesce(1 + + ceil(reltuples/floor((bs-pageopqdata-pagehdr)*fillfactor/(100*(4+nulldatahdrwidth)::float))), 0 + ) AS est_pages_ff, + bs, nspname, tblname, idxname, relpages, fillfactor, is_na + -- , pgstatindex(idxoid) AS pst, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples -- (DEBUG INFO) + FROM ( + SELECT maxalign, bs, nspname, tblname, idxname, reltuples, relpages, idxoid, fillfactor, + ( index_tuple_hdr_bm + + maxalign - CASE -- Add padding to the index tuple header to align on MAXALIGN + WHEN index_tuple_hdr_bm%maxalign = 0 THEN maxalign + ELSE index_tuple_hdr_bm%maxalign + END + + nulldatawidth + maxalign - CASE -- Add padding to the data to align on MAXALIGN + WHEN nulldatawidth = 0 THEN 0 + WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign + ELSE nulldatawidth::integer%maxalign + END + )::numeric AS nulldatahdrwidth, pagehdr, pageopqdata, is_na + -- , index_tuple_hdr_bm, nulldatawidth -- (DEBUG INFO) + FROM ( + SELECT n.nspname, i.tblname, i.idxname, i.reltuples, i.relpages, + i.idxoid, i.fillfactor, current_setting(''block_size'')::numeric AS bs, + CASE -- MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?) + WHEN version() ~ ''mingw32'' OR version() ~ ''64-bit|x86_64|ppc64|ia64|amd64'' THEN 8 + ELSE 4 + END AS maxalign, + /* per page header, fixed size: 20 for 7.X, 24 for others */ + 24 AS pagehdr, + /* per page btree opaque data */ + 16 AS pageopqdata, + /* per tuple header: add IndexAttributeBitMapData if some cols are null-able */ + CASE WHEN max(coalesce(s.null_frac,0)) = 0 + THEN 8 -- IndexTupleData size + ELSE 8 + (( 32 + 8 - 1 ) / 8) -- IndexTupleData size + IndexAttributeBitMapData size ( max num filed per index + 8 - 1 /8) + END AS index_tuple_hdr_bm, + /* data len: we remove null values save space using it fractionnal part from stats */ + sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) AS nulldatawidth, + max( CASE WHEN i.atttypid = ''pg_catalog.name''::regtype THEN 1 ELSE 0 END ) > 0 AS is_na + FROM ( + SELECT ct.relname AS tblname, ct.relnamespace, ic.idxname, ic.attpos, ic.indkey, ic.indkey[ic.attpos], ic.reltuples, ic.relpages, ic.tbloid, ic.idxoid, ic.fillfactor, + coalesce(a1.attnum, a2.attnum) AS attnum, coalesce(a1.attname, a2.attname) AS attname, coalesce(a1.atttypid, a2.atttypid) AS atttypid, + CASE WHEN a1.attnum IS NULL + THEN ic.idxname + ELSE ct.relname + END AS attrelname + FROM ( + SELECT idxname, reltuples, relpages, tbloid, idxoid, fillfactor, indkey, + pg_catalog.generate_series(1,indnatts) AS attpos + FROM ( + SELECT ci.relname AS idxname, ci.reltuples, ci.relpages, i.indrelid AS tbloid, + i.indexrelid AS idxoid, + coalesce(substring( + array_to_string(ci.reloptions, '' '') + from ''fillfactor=([0-9]+)'')::smallint, 90) AS fillfactor, + i.indnatts, + pg_catalog.string_to_array(pg_catalog.textin( + pg_catalog.int2vectorout(i.indkey)),'' '')::int[] AS indkey + FROM pg_catalog.pg_index i + JOIN pg_catalog.pg_class ci ON ci.oid = i.indexrelid + WHERE ci.relam=(SELECT oid FROM pg_am WHERE amname = ''btree'') + AND ci.relpages > 0 + ) AS idx_data + ) AS ic + JOIN pg_catalog.pg_class ct ON ct.oid = ic.tbloid + LEFT JOIN pg_catalog.pg_attribute a1 ON + ic.indkey[ic.attpos] <> 0 + AND a1.attrelid = ic.tbloid + AND a1.attnum = ic.indkey[ic.attpos] + LEFT JOIN pg_catalog.pg_attribute a2 ON + ic.indkey[ic.attpos] = 0 + AND a2.attrelid = ic.idxoid + AND a2.attnum = ic.attpos + ) i + JOIN pg_catalog.pg_namespace n ON n.oid = i.relnamespace + JOIN pg_catalog.pg_stats s ON s.schemaname = n.nspname + AND s.tablename = i.attrelname + AND s.attname = i.attname + GROUP BY 1,2,3,4,5,6,7,8,9,10,11 + ) AS rows_data_stats + ) AS rows_hdr_pdg_stats +) AS relation_stats +ORDER BY nspname, tblname, idxname; + +CREATE VIEW table_bloat AS /* WARNING: executed with a non-superuser role, the query inspect only tables and materialized view (9.3+) you are granted to read. +* This query is compatible with PostgreSQL 9.0 and more +*/ +SELECT current_database(), schemaname, tblname, bs*tblpages AS real_size, + (tblpages-est_tblpages)*bs AS extra_size, + CASE WHEN tblpages > 0 AND tblpages - est_tblpages > 0 + THEN 100 * (tblpages - est_tblpages)/tblpages::float + ELSE 0 + END AS extra_pct, fillfactor, + CASE WHEN tblpages - est_tblpages_ff > 0 + THEN (tblpages-est_tblpages_ff)*bs + ELSE 0 + END AS bloat_size, + CASE WHEN tblpages > 0 AND tblpages - est_tblpages_ff > 0 + THEN 100 * (tblpages - est_tblpages_ff)/tblpages::float + ELSE 0 + END AS bloat_pct, is_na + -- , tpl_hdr_size, tpl_data_size, (pst).free_percent + (pst).dead_tuple_percent AS real_frag -- (DEBUG INFO) +FROM ( + SELECT ceil( reltuples / ( (bs-page_hdr)/tpl_size ) ) + ceil( toasttuples / 4 ) AS est_tblpages, + ceil( reltuples / ( (bs-page_hdr)*fillfactor/(tpl_size*100) ) ) + ceil( toasttuples / 4 ) AS est_tblpages_ff, + tblpages, fillfactor, bs, tblid, schemaname, tblname, heappages, toastpages, is_na + -- , tpl_hdr_size, tpl_data_size, pgstattuple(tblid) AS pst -- (DEBUG INFO) + FROM ( + SELECT + ( 4 + tpl_hdr_size + tpl_data_size + (2*ma) + - CASE WHEN tpl_hdr_size%ma = 0 THEN ma ELSE tpl_hdr_size%ma END + - CASE WHEN ceil(tpl_data_size)::int%ma = 0 THEN ma ELSE ceil(tpl_data_size)::int%ma END + ) AS tpl_size, bs - page_hdr AS size_per_block, (heappages + toastpages) AS tblpages, heappages, + toastpages, reltuples, toasttuples, bs, page_hdr, tblid, schemaname, tblname, fillfactor, is_na + -- , tpl_hdr_size, tpl_data_size + FROM ( + SELECT + tbl.oid AS tblid, ns.nspname AS schemaname, tbl.relname AS tblname, tbl.reltuples, + tbl.relpages AS heappages, coalesce(toast.relpages, 0) AS toastpages, + coalesce(toast.reltuples, 0) AS toasttuples, + coalesce(substring( + array_to_string(tbl.reloptions, '' '') + FROM ''fillfactor=([0-9]+)'')::smallint, 100) AS fillfactor, + current_setting(''block_size'')::numeric AS bs, + CASE WHEN version()~''mingw32'' OR version()~''64-bit|x86_64|ppc64|ia64|amd64'' THEN 8 ELSE 4 END AS ma, + 24 AS page_hdr, + 23 + CASE WHEN MAX(coalesce(s.null_frac,0)) > 0 THEN ( 7 + count(s.attname) ) / 8 ELSE 0::int END + + CASE WHEN bool_or(att.attname = ''oid'' and att.attnum < 0) THEN 4 ELSE 0 END AS tpl_hdr_size, + sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 0) ) AS tpl_data_size, + bool_or(att.atttypid = ''pg_catalog.name''::regtype) + OR sum(CASE WHEN att.attnum > 0 THEN 1 ELSE 0 END) <> count(s.attname) AS is_na + FROM pg_attribute AS att + JOIN pg_class AS tbl ON att.attrelid = tbl.oid + JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace + LEFT JOIN pg_stats AS s ON s.schemaname=ns.nspname + AND s.tablename = tbl.relname AND s.inherited=false AND s.attname=att.attname + LEFT JOIN pg_class AS toast ON tbl.reltoastrelid = toast.oid + WHERE NOT att.attisdropped + AND tbl.relkind in (''r'',''m'') + GROUP BY 1,2,3,4,5,6,7,8,9,10 + ORDER BY 2,3 + ) AS s + ) AS s2 +) AS s3 +-- WHERE NOT is_na +-- AND tblpages*((pst).free_percent + (pst).dead_tuple_percent)::float4/100 >= 1 +ORDER BY schemaname, tblname; + +-- From https://wiki.postgresql.org/wiki/Lock_dependency_information + +CREATE OR REPLACE VIEW blocking_pid_tree AS +WITH RECURSIVE + lock_composite(requested, current) AS (VALUES + (''AccessShareLock''::text, ''AccessExclusiveLock''::text), + (''RowShareLock''::text, ''ExclusiveLock''::text), + (''RowShareLock''::text, ''AccessExclusiveLock''::text), + (''RowExclusiveLock''::text, ''ShareLock''::text), + (''RowExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''RowExclusiveLock''::text, ''ExclusiveLock''::text), + (''RowExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ShareLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''ShareLock''::text, ''RowExclusiveLock''::text), + (''ShareLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ShareLock''::text, ''ShareRowExclusiveLock''::text), + (''ShareLock''::text, ''ExclusiveLock''::text), + (''ShareLock''::text, ''AccessExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''RowExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''ShareLock''::text), + (''ShareRowExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''ExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''ExclusiveLock''::text, ''RowShareLock''::text), + (''ExclusiveLock''::text, ''RowExclusiveLock''::text), + (''ExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ExclusiveLock''::text, ''ShareLock''::text), + (''ExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''ExclusiveLock''::text, ''ExclusiveLock''::text), + (''ExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''AccessShareLock''::text), + (''AccessExclusiveLock''::text, ''RowShareLock''::text), + (''AccessExclusiveLock''::text, ''RowExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''ShareLock''::text), + (''AccessExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''ExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''AccessExclusiveLock''::text) + ) +, lock AS ( + SELECT pid, + virtualtransaction, + granted, + mode, + (locktype, + CASE locktype + WHEN ''relation'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text) + WHEN ''extend'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text) + WHEN ''page'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text, ''page#''||page::text) + WHEN ''tuple'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text, ''page#''||page::text, ''tuple#''||tuple::text) + WHEN ''transactionid'' THEN transactionid::text + WHEN ''virtualxid'' THEN virtualxid::text + WHEN ''object'' THEN concat_ws('';'', ''class:''||classid::regclass::text, ''objid:''||objid, ''col#''||objsubid) + ELSE concat(''db:''||datname) + END::text) AS target + FROM pg_catalog.pg_locks + LEFT JOIN pg_catalog.pg_database ON (pg_database.oid = pg_locks.database) + ) +, waiting_lock AS ( + SELECT + blocker.pid AS blocker_pid, + blocked.pid AS pid, + concat(blocked.mode,blocked.target) AS lock_target + FROM lock blocker + JOIN lock blocked + ON ( NOT blocked.granted + AND blocker.granted + AND blocked.pid != blocker.pid + AND blocked.target IS NOT DISTINCT FROM blocker.target) + JOIN lock_composite c ON (c.requested = blocked.mode AND c.current = blocker.mode) + ) +, acquired_lock AS ( + WITH waiting AS ( + SELECT lock_target, count(lock_target) AS wait_count FROM waiting_lock GROUP BY lock_target + ) + SELECT + pid, + array_agg(concat(mode,target,'' + ''||wait_count) ORDER BY wait_count DESC NULLS LAST) AS locks_acquired + FROM lock + LEFT JOIN waiting ON waiting.lock_target = concat(mode,target) + WHERE granted + GROUP BY pid + ) +, blocking_lock AS ( + SELECT + ARRAY[date_part(''epoch'', query_start)::int, pid] AS seq, + 0::int AS depth, + -1::int AS blocker_pid, + pid, + concat(''Connect: '',usename,'' '',datname,'' '',coalesce(host(client_addr)||'':''||client_port, ''local'') + , E''\nSQL: '',replace(substr(coalesce(query,''N/A''), 1, 60), E''\n'', '' '') + , E''\nAcquired:\n '' + , array_to_string(locks_acquired[1:5] || + CASE WHEN array_upper(locks_acquired,1) > 5 + THEN ''... ''||(array_upper(locks_acquired,1) - 5)::text||'' more ...'' + END, + E''\n '') + ) AS lock_info, + concat(to_char(query_start, CASE WHEN age(query_start) > ''24h'' THEN ''Day DD Mon'' ELSE ''HH24:MI:SS'' END),E'' started\n'' + ,CASE WHEN wait_event IS NOT NULL THEN ''waiting'' ELSE state END,E''\n'' + ,date_trunc(''second'',age(now(),query_start)),'' ago'' + ) AS lock_state + FROM acquired_lock blocker + LEFT JOIN pg_stat_activity act USING (pid) + WHERE EXISTS + (SELECT ''x'' FROM waiting_lock blocked WHERE blocked.blocker_pid = blocker.pid) + AND NOT EXISTS + (SELECT ''x'' FROM waiting_lock blocked WHERE blocked.pid = blocker.pid) +UNION ALL + SELECT + blocker.seq || blocked.pid, + blocker.depth + 1, + blocker.pid, + blocked.pid, + concat(''Connect: '',usename,'' '',datname,'' '',coalesce(host(client_addr)||'':''||client_port, ''local'') + , E''\nSQL: '',replace(substr(coalesce(query,''N/A''), 1, 60), E''\n'', '' '') + , E''\nWaiting: '',blocked.lock_target + , CASE WHEN locks_acquired IS NOT NULL + THEN E''\nAcquired:\n '' || + array_to_string(locks_acquired[1:5] || + CASE WHEN array_upper(locks_acquired,1) > 5 + THEN ''... ''||(array_upper(locks_acquired,1) - 5)::text||'' more ...'' + END, + E''\n '') + END + ) AS lock_info, + concat(to_char(query_start, CASE WHEN age(query_start) > ''24h'' THEN ''Day DD Mon'' ELSE ''HH24:MI:SS'' END),E'' started\n'' + ,CASE WHEN wait_event IS NOT NULL THEN ''waiting'' ELSE state END,E''\n'' + ,date_trunc(''second'',age(now(),query_start)),'' ago'' + ) AS lock_state + FROM blocking_lock blocker + JOIN waiting_lock blocked + ON (blocked.blocker_pid = blocker.pid) + LEFT JOIN pg_stat_activity act ON (act.pid = blocked.pid) + LEFT JOIN acquired_lock acq ON (acq.pid = blocked.pid) + WHERE blocker.depth < 5 + ) +SELECT concat(lpad(''=> '', 4*depth, '' ''),pid::text) AS "PID" +, lock_info AS "Lock Info" +, lock_state AS "State" +FROM blocking_lock +ORDER BY seq; + + +-- From https://wiki.postgresql.org/wiki/Index_Maintenance + +CREATE VIEW duplicate_indexes AS SELECT pg_size_pretty(sum(pg_relation_size(idx))::bigint) as size, + (array_agg(idx))[1] as idx1, (array_agg(idx))[2] as idx2, + (array_agg(idx))[3] as idx3, (array_agg(idx))[4] as idx4 +FROM ( + SELECT indexrelid::regclass as idx, (indrelid::text ||E''\n''|| indclass::text ||E''\n''|| indkey::text ||E''\n''|| + coalesce(indexprs::text,'''')||E''\n'' || coalesce(indpred::text,'''')) as key + FROM pg_index) sub +GROUP BY key HAVING count(*)>1 +ORDER BY sum(pg_relation_size(idx)) DESC; + +-- From https://wiki.postgresql.org/wiki/Disk_Usage + +CREATE VIEW table_sizes AS WITH RECURSIVE pg_inherit(inhrelid, inhparent) AS + (select inhrelid, inhparent + FROM pg_inherits + UNION + SELECT child.inhrelid, parent.inhparent + FROM pg_inherit child, pg_inherits parent + WHERE child.inhparent = parent.inhrelid), +pg_inherit_short AS (SELECT * FROM pg_inherit WHERE inhparent NOT IN (SELECT inhrelid FROM pg_inherit)) +SELECT table_schema + , TABLE_NAME + , row_estimate + , pg_size_pretty(total_bytes) AS total + , pg_size_pretty(index_bytes) AS INDEX + , pg_size_pretty(toast_bytes) AS toast + , pg_size_pretty(table_bytes) AS TABLE + , total_bytes::float8 / sum(total_bytes) OVER () AS total_size_share + FROM ( + SELECT *, total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes + FROM ( + SELECT c.oid + , nspname AS table_schema + , relname AS TABLE_NAME + , SUM(c.reltuples) OVER (partition BY parent) AS row_estimate + , SUM(pg_total_relation_size(c.oid)) OVER (partition BY parent) AS total_bytes + , SUM(pg_indexes_size(c.oid)) OVER (partition BY parent) AS index_bytes + , SUM(pg_total_relation_size(reltoastrelid)) OVER (partition BY parent) AS toast_bytes + , parent + FROM ( + SELECT pg_class.oid + , reltuples + , relname + , relnamespace + , pg_class.reltoastrelid + , COALESCE(inhparent, pg_class.oid) parent + FROM pg_class + LEFT JOIN pg_inherit_short ON inhrelid = oid + WHERE relkind IN (''r'', ''p'') + ) c + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace + ) a + WHERE oid = parent +) a +ORDER BY total_bytes DESC; + +-- From: https://wiki.postgresql.org/wiki/Index_Maintenance + +CREATE VIEW index_usage AS SELECT + t.schemaname, + t.tablename, + c.reltuples::bigint AS num_rows, + pg_size_pretty(pg_relation_size(c.oid)) AS table_size, + psai.indexrelname AS index_name, + pg_size_pretty(pg_relation_size(i.indexrelid)) AS index_size, + CASE WHEN i.indisunique THEN ''Y'' ELSE ''N'' END AS "unique", + psai.idx_scan AS number_of_scans, + psai.idx_tup_read AS tuples_read, + psai.idx_tup_fetch AS tuples_fetched +FROM + pg_tables t + LEFT JOIN pg_class c ON t.tablename = c.relname + LEFT JOIN pg_index i ON c.oid = i.indrelid + LEFT JOIN pg_stat_all_indexes psai ON i.indexrelid = psai.indexrelid +WHERE + t.schemaname NOT IN (''pg_catalog'', ''information_schema'') +ORDER BY 1, 2; + +-- From: https://blog.devgenius.io/top-useful-sql-queries-for-postgresql-35ff3355d265 + +CREATE VIEW database_sizes AS SELECT pg_database.datname, + pg_size_pretty(pg_database_size(pg_database.datname)) AS size +FROM pg_database +ORDER BY pg_database_size(pg_database.datname) DESC; + +CREATE VIEW schema_sizes AS SELECT A.schemaname, + pg_size_pretty (SUM(pg_relation_size(C.oid))) as table, + pg_size_pretty (SUM(pg_total_relation_size(C.oid)-pg_relation_size(C.oid))) as index, + pg_size_pretty (SUM(pg_total_relation_size(C.oid))) as table_index, + SUM(n_live_tup) +FROM pg_class C +LEFT JOIN pg_namespace N ON (N.oid = C .relnamespace) +INNER JOIN pg_stat_user_tables A ON C.relname = A.relname +WHERE nspname NOT IN (''pg_catalog'', ''information_schema'') +AND C .relkind <> ''i'' +AND nspname !~ ''^pg_toast'' +GROUP BY A.schemaname; + +CREATE VIEW last_vacuum_analyze AS SELECT relname, + last_vacuum, + last_autovacuum + n_mod_since_analyze, + last_analyze, + last_autoanalyze, + analyze_count, + autoanalyze_count + FROM pg_stat_user_tables; + +CREATE VIEW table_row_estimates AS SELECT + schemaname, + relname, + n_live_tup +FROM + pg_stat_user_tables +ORDER BY + n_live_tup DESC; + +-- postgres-meta queries as views from: https://github.com/supabase/postgres-meta + +CREATE VIEW pgmeta_columns AS SELECT + c.oid :: int8 AS table_id, + nc.nspname AS schema, + c.relname AS table, + (c.oid || ''.'' || a.attnum) AS id, + a.attnum AS ordinal_position, + a.attname AS name, + CASE + WHEN a.atthasdef THEN pg_get_expr(ad.adbin, ad.adrelid) + ELSE NULL + END AS default_value, + CASE + WHEN t.typtype = ''d'' THEN CASE + WHEN bt.typelem <> 0 :: oid + AND bt.typlen = -1 THEN ''ARRAY'' + WHEN nbt.nspname = ''pg_catalog'' THEN format_type(t.typbasetype, NULL) + ELSE ''USER-DEFINED'' + END + ELSE CASE + WHEN t.typelem <> 0 :: oid + AND t.typlen = -1 THEN ''ARRAY'' + WHEN nt.nspname = ''pg_catalog'' THEN format_type(a.atttypid, NULL) + ELSE ''USER-DEFINED'' + END + END AS data_type, + COALESCE(bt.typname, t.typname) AS format, + a.attidentity IN (''a'', ''d'') AS is_identity, + CASE + a.attidentity + WHEN ''a'' THEN ''ALWAYS'' + WHEN ''d'' THEN ''BY DEFAULT'' + ELSE NULL + END AS identity_generation, + a.attgenerated IN (''s'') AS is_generated, + NOT ( + a.attnotnull + OR t.typtype = ''d'' AND t.typnotnull + ) AS is_nullable, + ( + c.relkind IN (''r'', ''p'') + OR c.relkind IN (''v'', ''f'') AND pg_column_is_updatable(c.oid, a.attnum, FALSE) + ) AS is_updatable, + uniques.table_id IS NOT NULL AS is_unique, + array_to_json( + array( + SELECT + enumlabel + FROM + pg_catalog.pg_enum enums + WHERE + enums.enumtypid = coalesce(bt.oid, t.oid) + OR enums.enumtypid = coalesce(bt.typelem, t.typelem) + ORDER BY + enums.enumsortorder + ) + ) AS enums, + col_description(c.oid, a.attnum) AS comment +FROM + pg_attribute a + LEFT JOIN pg_attrdef ad ON a.attrelid = ad.adrelid + AND a.attnum = ad.adnum + JOIN ( + pg_class c + JOIN pg_namespace nc ON c.relnamespace = nc.oid + ) ON a.attrelid = c.oid + JOIN ( + pg_type t + JOIN pg_namespace nt ON t.typnamespace = nt.oid + ) ON a.atttypid = t.oid + LEFT JOIN ( + pg_type bt + JOIN pg_namespace nbt ON bt.typnamespace = nbt.oid + ) ON t.typtype = ''d'' + AND t.typbasetype = bt.oid + LEFT JOIN ( + SELECT + conrelid AS table_id, + conkey[1] AS ordinal_position + FROM pg_catalog.pg_constraint + WHERE contype = ''u'' AND cardinality(conkey) = 1 + ) AS uniques ON uniques.table_id = c.oid AND uniques.ordinal_position = a.attnum +WHERE + NOT pg_is_other_temp_schema(nc.oid) + AND a.attnum > 0 + AND NOT a.attisdropped + AND (c.relkind IN (''r'', ''v'', ''m'', ''f'', ''p'')) + AND ( + pg_has_role(c.relowner, ''USAGE'') + OR has_column_privilege( + c.oid, + a.attnum, + ''SELECT, INSERT, UPDATE, REFERENCES'' + ) + ); + +CREATE VIEW pgmeta_config AS SELECT + name, + setting, + category, + TRIM(split_part(category, ''/'', 1)) AS group, + TRIM(split_part(category, ''/'', 2)) AS subgroup, + unit, + short_desc, + extra_desc, + context, + vartype, + source, + min_val, + max_val, + enumvals, + boot_val, + reset_val, + sourcefile, + sourceline, + pending_restart +FROM + pg_settings +ORDER BY + category, + name; + +CREATE VIEW pgmeta_extensions AS SELECT + e.name, + n.nspname AS schema, + e.default_version, + x.extversion AS installed_version, + e.comment +FROM + pg_available_extensions() e(name, default_version, comment) + LEFT JOIN pg_extension x ON e.name = x.extname + LEFT JOIN pg_namespace n ON x.extnamespace = n.oid; + +CREATE VIEW pgmeta_foreign_tables AS SELECT + c.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS name, + obj_description(c.oid) AS comment +FROM + pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE + c.relkind = ''f''; + +CREATE VIEW pgmeta_functions AS with functions as ( + select + *, + -- proargmodes is null when all arg modes are IN + coalesce( + p.proargmodes, + array_fill(''i''::text, array[cardinality(coalesce(p.proallargtypes, p.proargtypes))]) + ) as arg_modes, + -- proargnames is null when all args are unnamed + coalesce( + p.proargnames, + array_fill(''''::text, array[cardinality(coalesce(p.proallargtypes, p.proargtypes))]) + ) as arg_names, + -- proallargtypes is null when all arg modes are IN + coalesce(p.proallargtypes, p.proargtypes) as arg_types, + array_cat( + array_fill(false, array[pronargs - pronargdefaults]), + array_fill(true, array[pronargdefaults])) as arg_has_defaults + from + pg_proc as p + where + p.prokind = ''f'' +) +select + f.oid::int8 as id, + n.nspname as schema, + f.proname as name, + l.lanname as language, + case + when l.lanname = ''internal'' then '''' + else f.prosrc + end as definition, + case + when l.lanname = ''internal'' then f.prosrc + else pg_get_functiondef(f.oid) + end as complete_statement, + coalesce(f_args.args, ''[]'') as args, + pg_get_function_arguments(f.oid) as argument_types, + pg_get_function_identity_arguments(f.oid) as identity_argument_types, + f.prorettype::int8 as return_type_id, + pg_get_function_result(f.oid) as return_type, + nullif(rt.typrelid::int8, 0) as return_type_relation_id, + f.proretset as is_set_returning_function, + case + when f.provolatile = ''i'' then ''IMMUTABLE'' + when f.provolatile = ''s'' then ''STABLE'' + when f.provolatile = ''v'' then ''VOLATILE'' + end as behavior, + f.prosecdef as security_definer, + f_config.config_params as config_params +from + functions f + left join pg_namespace n on f.pronamespace = n.oid + left join pg_language l on f.prolang = l.oid + left join pg_type rt on rt.oid = f.prorettype + left join ( + select + oid, + jsonb_object_agg(param, value) filter (where param is not null) as config_params + from + ( + select + oid, + (string_to_array(unnest(proconfig), ''=''))[1] as param, + (string_to_array(unnest(proconfig), ''=''))[2] as value + from + functions + ) as t + group by + oid + ) f_config on f_config.oid = f.oid + left join ( + select + oid, + jsonb_agg(jsonb_build_object( + ''mode'', t2.mode, + ''name'', name, + ''type_id'', type_id, + ''has_default'', has_default + )) as args + from + ( + select + oid, + unnest(arg_modes) as mode, + unnest(arg_names) as name, + unnest(arg_types)::int8 as type_id, + unnest(arg_has_defaults) as has_default + from + functions + ) as t1, + lateral ( + select + case + when t1.mode = ''i'' then ''in'' + when t1.mode = ''o'' then ''out'' + when t1.mode = ''b'' then ''inout'' + when t1.mode = ''v'' then ''variadic'' + else ''table'' + end as mode + ) as t2 + group by + t1.oid + ) f_args on f_args.oid = f.oid; + +CREATE VIEW pgmeta_materialized_views AS select + c.oid::int8 as id, + n.nspname as schema, + c.relname as name, + c.relispopulated as is_populated, + obj_description(c.oid) as comment +from + pg_class c + join pg_namespace n on n.oid = c.relnamespace +where + c.relkind = ''m''; + +CREATE VIEW pgmeta_policies AS SELECT + pol.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS table, + c.oid :: int8 AS table_id, + pol.polname AS name, + CASE + WHEN pol.polpermissive THEN ''PERMISSIVE'' :: text + ELSE ''RESTRICTIVE'' :: text + END AS action, + CASE + WHEN pol.polroles = ''{0}'' :: oid [] THEN array_to_json( + string_to_array(''public'' :: text, '''' :: text) :: name [] + ) + ELSE array_to_json( + ARRAY( + SELECT + pg_roles.rolname + FROM + pg_roles + WHERE + pg_roles.oid = ANY (pol.polroles) + ORDER BY + pg_roles.rolname + ) + ) + END AS roles, + CASE + pol.polcmd + WHEN ''r'' :: "char" THEN ''SELECT'' :: text + WHEN ''a'' :: "char" THEN ''INSERT'' :: text + WHEN ''w'' :: "char" THEN ''UPDATE'' :: text + WHEN ''d'' :: "char" THEN ''DELETE'' :: text + WHEN ''*'' :: "char" THEN ''ALL'' :: text + ELSE NULL :: text + END AS command, + pg_get_expr(pol.polqual, pol.polrelid) AS definition, + pg_get_expr(pol.polwithcheck, pol.polrelid) AS check +FROM + pg_policy pol + JOIN pg_class c ON c.oid = pol.polrelid + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace; + +CREATE VIEW pgmeta_primary_keys AS SELECT + n.nspname AS schema, + c.relname AS table_name, + a.attname AS name, + c.oid :: int8 AS table_id +FROM + pg_index i, + pg_class c, + pg_attribute a, + pg_namespace n +WHERE + i.indrelid = c.oid + AND c.relnamespace = n.oid + AND a.attrelid = c.oid + AND a.attnum = ANY (i.indkey) + AND i.indisprimary; + +CREATE VIEW pgmeta_publications AS SELECT + p.oid :: int8 AS id, + p.pubname AS name, + p.pubowner::regrole::text AS owner, + p.pubinsert AS publish_insert, + p.pubupdate AS publish_update, + p.pubdelete AS publish_delete, + p.pubtruncate AS publish_truncate, + CASE + WHEN p.puballtables THEN NULL + ELSE pr.tables + END AS tables +FROM + pg_catalog.pg_publication AS p + LEFT JOIN LATERAL ( + SELECT + COALESCE( + array_agg( + json_build_object( + ''id'', + c.oid :: int8, + ''name'', + c.relname, + ''schema'', + nc.nspname + ) + ), + ''{}'' + ) AS tables + FROM + pg_catalog.pg_publication_rel AS pr + JOIN pg_class AS c ON pr.prrelid = c.oid + join pg_namespace as nc on c.relnamespace = nc.oid + WHERE + pr.prpubid = p.oid + ) AS pr ON 1 = 1; + +CREATE VIEW pgmeta_relationships AS SELECT + c.oid :: int8 AS id, + c.conname AS constraint_name, + nsa.nspname AS source_schema, + csa.relname AS source_table_name, + sa.attname AS source_column_name, + nta.nspname AS target_table_schema, + cta.relname AS target_table_name, + ta.attname AS target_column_name +FROM + pg_constraint c + JOIN ( + pg_attribute sa + JOIN pg_class csa ON sa.attrelid = csa.oid + JOIN pg_namespace nsa ON csa.relnamespace = nsa.oid + ) ON sa.attrelid = c.conrelid + AND sa.attnum = ANY (c.conkey) + JOIN ( + pg_attribute ta + JOIN pg_class cta ON ta.attrelid = cta.oid + JOIN pg_namespace nta ON cta.relnamespace = nta.oid + ) ON ta.attrelid = c.confrelid + AND ta.attnum = ANY (c.confkey) +WHERE + c.contype = ''f''; + +CREATE VIEW pgmeta_roles AS -- TODO: Consider using pg_authid vs. pg_roles for unencrypted password field +SELECT + oid :: int8 AS id, + rolname AS name, + rolsuper AS is_superuser, + rolcreatedb AS can_create_db, + rolcreaterole AS can_create_role, + rolinherit AS inherit_role, + rolcanlogin AS can_login, + rolreplication AS is_replication_role, + rolbypassrls AS can_bypass_rls, + ( + SELECT + COUNT(*) + FROM + pg_stat_activity + WHERE + pg_roles.rolname = pg_stat_activity.usename + ) AS active_connections, + CASE WHEN rolconnlimit = -1 THEN current_setting(''max_connections'') :: int8 + ELSE rolconnlimit + END AS connection_limit, + rolpassword AS password, + rolvaliduntil AS valid_until, + rolconfig AS config +FROM + pg_roles; + +CREATE VIEW pgmeta_schemas AS select + n.oid::int8 as id, + n.nspname as name, + u.rolname as owner +from + pg_namespace n, + pg_roles u +where + n.nspowner = u.oid + and ( + pg_has_role(n.nspowner, ''USAGE'') + or has_schema_privilege(n.oid, ''CREATE, USAGE'') + ) + and not pg_catalog.starts_with(n.nspname, ''pg_temp_'') + and not pg_catalog.starts_with(n.nspname, ''pg_toast_temp_''); + +CREATE VIEW pgmeta_tables AS SELECT + c.oid :: int8 AS id, + nc.nspname AS schema, + c.relname AS name, + c.relrowsecurity AS rls_enabled, + c.relforcerowsecurity AS rls_forced, + CASE + WHEN c.relreplident = ''d'' THEN ''DEFAULT'' + WHEN c.relreplident = ''i'' THEN ''INDEX'' + WHEN c.relreplident = ''f'' THEN ''FULL'' + ELSE ''NOTHING'' + END AS replica_identity, + pg_total_relation_size(format(''%I.%I'', nc.nspname, c.relname)) :: int8 AS bytes, + pg_size_pretty( + pg_total_relation_size(format(''%I.%I'', nc.nspname, c.relname)) + ) AS size, + pg_stat_get_live_tuples(c.oid) AS live_rows_estimate, + pg_stat_get_dead_tuples(c.oid) AS dead_rows_estimate, + obj_description(c.oid) AS comment +FROM + pg_namespace nc + JOIN pg_class c ON nc.oid = c.relnamespace +WHERE + c.relkind IN (''r'', ''p'') + AND NOT pg_is_other_temp_schema(nc.oid) + AND ( + pg_has_role(c.relowner, ''USAGE'') + OR has_table_privilege( + c.oid, + ''SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER'' + ) + OR has_any_column_privilege(c.oid, ''SELECT, INSERT, UPDATE, REFERENCES'') + ); + + +CREATE VIEW pgmeta_triggers AS SELECT + pg_t.oid AS id, + pg_t.tgrelid AS table_id, + CASE + WHEN pg_t.tgenabled = ''D'' THEN ''DISABLED'' + WHEN pg_t.tgenabled = ''O'' THEN ''ORIGIN'' + WHEN pg_t.tgenabled = ''R'' THEN ''REPLICA'' + WHEN pg_t.tgenabled = ''A'' THEN ''ALWAYS'' + END AS enabled_mode, + ( + STRING_TO_ARRAY( + ENCODE(pg_t.tgargs, ''escape''), ''\000'' + ) + )[:pg_t.tgnargs] AS function_args, + is_t.trigger_name AS name, + is_t.event_object_table AS table, + is_t.event_object_schema AS schema, + is_t.action_condition AS condition, + is_t.action_orientation AS orientation, + is_t.action_timing AS activation, + ARRAY_AGG(is_t.event_manipulation)::text[] AS events, + pg_p.proname AS function_name, + pg_n.nspname AS function_schema +FROM + pg_trigger AS pg_t +JOIN + pg_class AS pg_c +ON pg_t.tgrelid = pg_c.oid +JOIN information_schema.triggers AS is_t +ON is_t.trigger_name = pg_t.tgname +AND pg_c.relname = is_t.event_object_table +JOIN pg_proc AS pg_p +ON pg_t.tgfoid = pg_p.oid +JOIN pg_namespace AS pg_n +ON pg_p.pronamespace = pg_n.oid +GROUP BY + pg_t.oid, + pg_t.tgrelid, + pg_t.tgenabled, + pg_t.tgargs, + pg_t.tgnargs, + is_t.trigger_name, + is_t.event_object_table, + is_t.event_object_schema, + is_t.action_condition, + is_t.action_orientation, + is_t.action_timing, + pg_p.proname, + pg_n.nspname; + + +CREATE VIEW pgmeta_types AS select + t.oid::int8 as id, + t.typname as name, + n.nspname as schema, + format_type (t.oid, null) as format, + coalesce(t_enums.enums, ''[]'') as enums, + coalesce(t_attributes.attributes, ''[]'') as attributes, + obj_description (t.oid, ''pg_type'') as comment +from + pg_type t + left join pg_namespace n on n.oid = t.typnamespace + left join ( + select + enumtypid, + jsonb_agg(enumlabel order by enumsortorder) as enums + from + pg_enum + group by + enumtypid + ) as t_enums on t_enums.enumtypid = t.oid + left join ( + select + oid, + jsonb_agg( + jsonb_build_object(''name'', a.attname, ''type_id'', a.atttypid::int8) + order by a.attnum asc + ) as attributes + from + pg_class c + join pg_attribute a on a.attrelid = c.oid + where + c.relkind = ''c'' and not a.attisdropped + group by + c.oid + ) as t_attributes on t_attributes.oid = t.typrelid +where + ( + t.typrelid = 0 + or ( + select + c.relkind = ''c'' + from + pg_class c + where + c.oid = t.typrelid + ) + ); + +CREATE VIEW pgmeta_version AS SELECT + version(), + current_setting(''server_version_num'') :: int8 AS version_number, + ( + SELECT + COUNT(*) AS active_connections + FROM + pg_stat_activity + ) AS active_connections, + current_setting(''max_connections'') :: int8 AS max_connections; + +CREATE VIEW pgmeta_views AS SELECT + c.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS name, + -- See definition of information_schema.views + (pg_relation_is_updatable(c.oid, false) & 20) = 20 AS is_updatable, + obj_description(c.oid) AS comment +FROM + pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE + c.relkind = ''v''; +', + ' +# michelp-adminpack + +A Trusted Language Extension containing a variety of useful databse admin queries. + +Postgres database admins are often faced with a variety of issues that +require them introspecting the database state. This extension +contains a mixed-bag of views that reflect useful information for +admins. + +## Installation + +Using dbdev: + +``` +postgres=> select dbdev.install(''michelp-adminpack''); + install +--------- + t +(1 row) + +postgres=> create schema adminpack; +CREATE SCHEMA +postgres=> create extension "michelp-adminpack" with schema adminpack; +CREATE EXTENSION +``` + +This will the extension into the `adminpack` schema, substitute with +some other schema if you want it to go somewhere else. + +## Index Bloat + +Index updates and querying can end up getting slower and slower over +time due to what''s called "index bloat". This is where frequent +updates and deletes can cause space in index data structures to go +unused. Understanding when this is happening is not obvious, so there +is a rather complex query you can run to detect it. + +`index_bloat` + +| Column | Type | +|------------------|------------------| +| current_database | name | +| schemaname | name | +| tblname | name | +| idxname | name | +| real_size | numeric | +| extra_size | numeric | +| extra_pct | double precision | +| fillfactor | integer | +| bloat_size | double precision | +| bloat_pct | double precision | +| is_na | boolean | + + +## Table Bloat + +Like index bloat, tables with high update and delete rates can also +end up containing lots of allocated but unused space. This query +shows which tables have the most bloat. + +`table_bloat` + +| Column | Type | +|------------------|------------------| +| current_database | name | +| schemaname | name | +| tblname | name | +| real_size | numeric | +| extra_size | double precision | +| extra_pct | double precision | +| fillfactor | integer | +| bloat_size | double precision | +| bloat_pct | double precision | +| is_na | boolean | + + +## Blocking PID Tree + +Postgres queries can block each other, and this blocking relationship +can form a tree, where A blocks B, which blocks C, and so on. This +query formats blocking queries into a tree structure so it''s easy to +see what query is causing the blockage. + +`blocking_pid_tree` + +| Column | Type | +|-----------|------| +| PID | text | +| Lock Info | text | +| State | text | + + +## Duplicate Indexes + +If a table contains duplicate indexes, then unecessary work is done +updating and storing them, this query will show up to 4 duplicate +indexes per table if they exist. + +`duplicate_indexes` + +| Column | Type | +|--------|----------| +| size | text | +| idx1 | regclass | +| idx2 | regclass | +| idx3 | regclass | +| idx4 | regclass | + + +## Table Sizes + +This view shows tables and their sizes. + +`table_sizes` + +| Column | Type | +|------------------|------------------| +| table_schema | name | +| table_name | name | +| row_estimate | real | +| total | text | +| index | text | +| toast | text | +| table | text | +| total_size_share | double precision | + + +## Schema Sizes + +This view shows schemas and their sizes, which is the sum of the sizes +of all the tables and indexes in the schema. + +`schema_sizes` + +| Column | Type | +|-------------|---------| +| schemaname | name | +| table | text | +| index | text | +| table_index | text | +| sum | numeric | + + +## Index Usage + +This view shows index size and usage. An unused index still needs to +be updated and that takes time an storage, so it''s a good candidate to +drop. + +`index_usage` + +| Column | Type | +|-----------------|--------| +| schemaname | name | +| tablename | name | +| num_rows | bigint | +| table_size | text | +| index_name | name | +| index_size | text | +| unique | text | +| number_of_scans | bigint | +| tuples_read | bigint | +| tuples_fetched | bigint | + + +## Last Vacuum Analyze + +This views shows the last time a table was vacuumed an analyzed. + +`last_vacuum_analyze` + +| Column | Type | +|---------------------|--------------------------| +| relname | name | +| last_vacuum | timestamp with time zone | +| n_mod_since_analyze | timestamp with time zone | +| last_analyze | timestamp with time zone | +| last_autoanalyze | timestamp with time zone | +| analyze_count | bigint | +| autoanalyze_count | bigint | + +## Table Row Estimates + +This view shows estimates for the number of rows in a table. This is +just an estimate and depends on up to date table statistics. + +`table_row_estimates` + +| Column | Type | +|------------|--------| +| schemaname | name | +| relname | name | +| n_live_tup | bigint | + +## PGMeta Columns + +This view shows infromation for the columns of tables in the system. + +`pgmeta_columns` + +| Column | Type | +|---------------------|----------| +| table_id | bigint | +| schema | name | +| table | name | +| id | text | +| ordinal_position | smallint | +| name | name | +| default_value | text | +| data_type | text | +| format | name | +| is_identity | boolean | +| identity_generation | text | +| is_generated | boolean | +| is_nullable | boolean | +| is_updatable | boolean | +| is_unique | boolean | +| enums | json | +| comment | text | + +## PGMeta Config + +This views shows the configuration of the database. + +`pgmeta_config` + +| Column | Type | +|-----------------|---------| +| name | text | +| setting | text | +| category | text | +| group | text | +| subgroup | text | +| unit | text | +| short_desc | text | +| extra_desc | text | +| context | text | +| vartype | text | +| source | text | +| min_val | text | +| max_val | text | +| enumvals | text[] | +| boot_val | text | +| reset_val | text | +| sourcefile | text | +| sourceline | integer | +| pending_restart | boolean | + +## PGMeta Extensions + +This view shows installed extensions in the database. + +`pgmeta_extensions` + +| Column | Type | +|-------------------|------| +| name | name | +| schema | name | +| default_version | text | +| installed_version | text | +| comment | text | + +## PGMeta Foreign Tables + +This view shows foreign tables in the database. + +`pgmeta_foreign_tables` + +| Column | Type | +|---------|--------| +| id | bigint | +| schema | name | +| name | name | +| comment | text | + +## PGMeta Functions + +This view shows functions in the database. + +`pgmeta_functions` + +| Column | Type | +|---------------------------|---------| +| id | bigint | +| schema | name | +| name | name | +| language | name | +| definition | text | +| complete_statement | text | +| args | jsonb | +| argument_types | text | +| identity_argument_types | text | +| return_type_id | bigint | +| return_type | text | +| return_type_relation_id | bigint | +| is_set_returning_function | boolean | +| behavior | text | +| security_definer | boolean | +| config_params | jsonb | + +## PGMeta Materialized Views + +This view shows materialized views in the database. + +`pgmeta_materialized_views` + +| Column | Type | +|--------------|---------| +| id | bigint | +| schema | name | +| name | name | +| is_populated | boolean | +| comment | text | + +## PGMeta Policies + +This view shows Row Level Security Policies in the database. + +`pgmeta_policies` + +| Column | Type | +|------------|--------| +| id | bigint | +| schema | name | +| table | name | +| table_id | bigint | +| name | name | +| action | text | +| roles | json | +| command | text | +| definition | text | +| check | text | + +## PGMeta Primary Keys + +This view shows primary keys in the database. + +`pgmeta_primary_keys` + +| Column | Type | +|------------|--------| +| schema | name | +| table_name | name | +| name | name | +| table_id | bigint | + +## PGMeta Publications + +This view shows logical replication publishers in the database. + +`pgmeta_publications` + +| Column | Type | +|------------------|---------| +| id | bigint | +| name | name | +| owner | text | +| publish_insert | boolean | +| publish_update | boolean | +| publish_delete | boolean | +| publish_truncate | boolean | +| tables | json[] | + +## PGMeta Relationships + +This view shows foreign key relationships in the database. + +`pgmeta_relationships` + +| Column | Type | +|---------------------|--------| +| id | bigint | +| constraint_name | name | +| source_schema | name | +| source_table_name | name | +| source_column_name | name | +| target_table_schema | name | +| target_table_name | name | +| target_column_name | name | + +## PGMeta Roles + +This view shows roles in the database system. Note that roles are +global objects and apply to all databases. + +`pgmeta_roles` + +| Column | Type | +|---------------------|--------------------------| +| id | bigint | +| name | name | +| is_superuser | boolean | +| can_create_db | boolean | +| can_create_role | boolean | +| inherit_role | boolean | +| can_login | boolean | +| is_replication_role | boolean | +| can_bypass_rls | boolean | +| active_connections | bigint | +| connection_limit | bigint | +| password | text | +| valid_until | timestamp with time zone | +| config | text[] | + +## PGMeta Schemas + +This view shows all schemas in the database. + +`pgmeta_schemas` + +| Column | Type | +|--------|--------| +| id | bigint | +| name | name | +| owner | name | + +## PGMeta Tables + +This view shows all tables in the database. + +`pgmeta_tables` + +| Column | Type | +|--------------------|---------| +| id | bigint | +| schema | name | +| name | name | +| rls_enabled | boolean | +| rls_forced | boolean | +| replica_identity | text | +| bytes | bigint | +| size | text | +| live_rows_estimate | bigint | +| dead_rows_estimate | bigint | +| comment | text | + +## PGMeta Triggers + +This view shows all triggers in the database. + +`pgmeta_triggers` + +| Column | Type | +|-----------------|-----------------------------------| +| id | oid | +| table_id | oid | +| enabled_mode | text | +| function_args | text[] | +| name | information_schema.sql_identifier | +| table | information_schema.sql_identifier | +| schema | information_schema.sql_identifier | +| condition | information_schema.character_data | +| orientation | information_schema.character_data | +| activation | information_schema.character_data | +| events | text[] | +| function_name | name | +| function_schema | name | + +## PGMeta Types + +This view shows all types in the database. + +`pgmeta_types` + +| Column | Type | +|------------|--------| +| id | bigint | +| name | name | +| schema | name | +| format | text | +| enums | jsonb | +| attributes | jsonb | +| comment | text | + +## PGMeta Version + +This view shows the current database version. + +`pgmeta_version` + +| Column | Type | +|--------------------|--------| +| version | text | +| version_number | bigint | +| active_connections | bigint | +| max_connections | bigint | + +## PGMeta Views + +This view shows all views in the database. + +`pgmeta_views` + +| Column | Type | +|--------------|---------| +| id | bigint | +| schema | name | +| name | name | +| is_updatable | boolean | +| comment | text | +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230404162614_michelp-adminpack_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230404162614_michelp-adminpack_80.snap new file mode 100644 index 000000000..e57df25d9 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230404162614_michelp-adminpack_80.snap @@ -0,0 +1,1587 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230404162614_michelp-adminpack.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'michelp', + 'adminpack', + 'A bunch of useful queries for DBA to manage and inspect databases', + false, + '{}' + ); + +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_name = 'michelp-adminpack' + ), + (0, 0, 1), + ' +-- From: https://github.com/ioguix/pgsql-bloat-estimation + +-- Copyright (c) 2015-2019, Jehan-Guillaume (ioguix) de Rorthais +-- All rights reserved. + +-- Redistribution and use in source and binary forms, with or without +-- modification, are permitted provided that the following conditions are met: + +-- * Redistributions of source code must retain the above copyright notice, this +-- list of conditions and the following disclaimer. + +-- * Redistributions in binary form must reproduce the above copyright notice, +-- this list of conditions and the following disclaimer in the documentation +-- and/or other materials provided with the distribution. + +-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +-- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +-- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +-- DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +-- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +-- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +-- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +-- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +-- OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +-- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-- WARNING: executed with a non-superuser role, the query inspect only index on tables you are granted to read. +-- WARNING: rows with is_na = ''t'' are known to have bad statistics ("name" type is not supported). +-- This query is compatible with PostgreSQL 8.2 and after +CREATE VIEW index_bloat AS SELECT current_database(), nspname AS schemaname, tblname, idxname, bs*(relpages)::bigint AS real_size, + bs*(relpages-est_pages)::bigint AS extra_size, + 100 * (relpages-est_pages)::float / relpages AS extra_pct, + fillfactor, + CASE WHEN relpages > est_pages_ff + THEN bs*(relpages-est_pages_ff) + ELSE 0 + END AS bloat_size, + 100 * (relpages-est_pages_ff)::float / relpages AS bloat_pct, + is_na + -- , 100-(pst).avg_leaf_density AS pst_avg_bloat, est_pages, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples, relpages -- (DEBUG INFO) +FROM ( + SELECT coalesce(1 + + ceil(reltuples/floor((bs-pageopqdata-pagehdr)/(4+nulldatahdrwidth)::float)), 0 -- ItemIdData size + computed avg size of a tuple (nulldatahdrwidth) + ) AS est_pages, + coalesce(1 + + ceil(reltuples/floor((bs-pageopqdata-pagehdr)*fillfactor/(100*(4+nulldatahdrwidth)::float))), 0 + ) AS est_pages_ff, + bs, nspname, tblname, idxname, relpages, fillfactor, is_na + -- , pgstatindex(idxoid) AS pst, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples -- (DEBUG INFO) + FROM ( + SELECT maxalign, bs, nspname, tblname, idxname, reltuples, relpages, idxoid, fillfactor, + ( index_tuple_hdr_bm + + maxalign - CASE -- Add padding to the index tuple header to align on MAXALIGN + WHEN index_tuple_hdr_bm%maxalign = 0 THEN maxalign + ELSE index_tuple_hdr_bm%maxalign + END + + nulldatawidth + maxalign - CASE -- Add padding to the data to align on MAXALIGN + WHEN nulldatawidth = 0 THEN 0 + WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign + ELSE nulldatawidth::integer%maxalign + END + )::numeric AS nulldatahdrwidth, pagehdr, pageopqdata, is_na + -- , index_tuple_hdr_bm, nulldatawidth -- (DEBUG INFO) + FROM ( + SELECT n.nspname, i.tblname, i.idxname, i.reltuples, i.relpages, + i.idxoid, i.fillfactor, current_setting(''block_size'')::numeric AS bs, + CASE -- MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?) + WHEN version() ~ ''mingw32'' OR version() ~ ''64-bit|x86_64|ppc64|ia64|amd64'' THEN 8 + ELSE 4 + END AS maxalign, + /* per page header, fixed size: 20 for 7.X, 24 for others */ + 24 AS pagehdr, + /* per page btree opaque data */ + 16 AS pageopqdata, + /* per tuple header: add IndexAttributeBitMapData if some cols are null-able */ + CASE WHEN max(coalesce(s.null_frac,0)) = 0 + THEN 8 -- IndexTupleData size + ELSE 8 + (( 32 + 8 - 1 ) / 8) -- IndexTupleData size + IndexAttributeBitMapData size ( max num filed per index + 8 - 1 /8) + END AS index_tuple_hdr_bm, + /* data len: we remove null values save space using it fractionnal part from stats */ + sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) AS nulldatawidth, + max( CASE WHEN i.atttypid = ''pg_catalog.name''::regtype THEN 1 ELSE 0 END ) > 0 AS is_na + FROM ( + SELECT ct.relname AS tblname, ct.relnamespace, ic.idxname, ic.attpos, ic.indkey, ic.indkey[ic.attpos], ic.reltuples, ic.relpages, ic.tbloid, ic.idxoid, ic.fillfactor, + coalesce(a1.attnum, a2.attnum) AS attnum, coalesce(a1.attname, a2.attname) AS attname, coalesce(a1.atttypid, a2.atttypid) AS atttypid, + CASE WHEN a1.attnum IS NULL + THEN ic.idxname + ELSE ct.relname + END AS attrelname + FROM ( + SELECT idxname, reltuples, relpages, tbloid, idxoid, fillfactor, indkey, + pg_catalog.generate_series(1,indnatts) AS attpos + FROM ( + SELECT ci.relname AS idxname, ci.reltuples, ci.relpages, i.indrelid AS tbloid, + i.indexrelid AS idxoid, + coalesce(substring( + array_to_string(ci.reloptions, '' '') + from ''fillfactor=([0-9]+)'')::smallint, 90) AS fillfactor, + i.indnatts, + pg_catalog.string_to_array(pg_catalog.textin( + pg_catalog.int2vectorout(i.indkey)),'' '')::int[] AS indkey + FROM pg_catalog.pg_index i + JOIN pg_catalog.pg_class ci ON ci.oid = i.indexrelid + WHERE ci.relam=(SELECT oid FROM pg_am WHERE amname = ''btree'') + AND ci.relpages > 0 + ) AS idx_data + ) AS ic + JOIN pg_catalog.pg_class ct ON ct.oid = ic.tbloid + LEFT JOIN pg_catalog.pg_attribute a1 ON + ic.indkey[ic.attpos] <> 0 + AND a1.attrelid = ic.tbloid + AND a1.attnum = ic.indkey[ic.attpos] + LEFT JOIN pg_catalog.pg_attribute a2 ON + ic.indkey[ic.attpos] = 0 + AND a2.attrelid = ic.idxoid + AND a2.attnum = ic.attpos + ) i + JOIN pg_catalog.pg_namespace n ON n.oid = i.relnamespace + JOIN pg_catalog.pg_stats s ON s.schemaname = n.nspname + AND s.tablename = i.attrelname + AND s.attname = i.attname + GROUP BY 1,2,3,4,5,6,7,8,9,10,11 + ) AS rows_data_stats + ) AS rows_hdr_pdg_stats +) AS relation_stats +ORDER BY nspname, tblname, idxname; + +CREATE VIEW table_bloat AS /* WARNING: executed with a non-superuser role, the query inspect only tables and materialized view (9.3+) you are granted to read. +* This query is compatible with PostgreSQL 9.0 and more +*/ +SELECT current_database(), schemaname, tblname, bs*tblpages AS real_size, + (tblpages-est_tblpages)*bs AS extra_size, + CASE WHEN tblpages > 0 AND tblpages - est_tblpages > 0 + THEN 100 * (tblpages - est_tblpages)/tblpages::float + ELSE 0 + END AS extra_pct, fillfactor, + CASE WHEN tblpages - est_tblpages_ff > 0 + THEN (tblpages-est_tblpages_ff)*bs + ELSE 0 + END AS bloat_size, + CASE WHEN tblpages > 0 AND tblpages - est_tblpages_ff > 0 + THEN 100 * (tblpages - est_tblpages_ff)/tblpages::float + ELSE 0 + END AS bloat_pct, is_na + -- , tpl_hdr_size, tpl_data_size, (pst).free_percent + (pst).dead_tuple_percent AS real_frag -- (DEBUG INFO) +FROM ( + SELECT ceil( reltuples / ( (bs-page_hdr)/tpl_size ) ) + ceil( toasttuples / 4 ) AS est_tblpages, + ceil( reltuples / ( (bs-page_hdr)*fillfactor/(tpl_size*100) ) ) + ceil( toasttuples / 4 ) AS est_tblpages_ff, + tblpages, fillfactor, bs, tblid, schemaname, tblname, heappages, toastpages, is_na + -- , tpl_hdr_size, tpl_data_size, pgstattuple(tblid) AS pst -- (DEBUG INFO) + FROM ( + SELECT + ( 4 + tpl_hdr_size + tpl_data_size + (2*ma) + - CASE WHEN tpl_hdr_size%ma = 0 THEN ma ELSE tpl_hdr_size%ma END + - CASE WHEN ceil(tpl_data_size)::int%ma = 0 THEN ma ELSE ceil(tpl_data_size)::int%ma END + ) AS tpl_size, bs - page_hdr AS size_per_block, (heappages + toastpages) AS tblpages, heappages, + toastpages, reltuples, toasttuples, bs, page_hdr, tblid, schemaname, tblname, fillfactor, is_na + -- , tpl_hdr_size, tpl_data_size + FROM ( + SELECT + tbl.oid AS tblid, ns.nspname AS schemaname, tbl.relname AS tblname, tbl.reltuples, + tbl.relpages AS heappages, coalesce(toast.relpages, 0) AS toastpages, + coalesce(toast.reltuples, 0) AS toasttuples, + coalesce(substring( + array_to_string(tbl.reloptions, '' '') + FROM ''fillfactor=([0-9]+)'')::smallint, 100) AS fillfactor, + current_setting(''block_size'')::numeric AS bs, + CASE WHEN version()~''mingw32'' OR version()~''64-bit|x86_64|ppc64|ia64|amd64'' THEN 8 ELSE 4 END AS ma, + 24 AS page_hdr, + 23 + CASE WHEN MAX(coalesce(s.null_frac,0)) > 0 THEN ( 7 + count(s.attname) ) / 8 ELSE 0::int END + + CASE WHEN bool_or(att.attname = ''oid'' and att.attnum < 0) THEN 4 ELSE 0 END AS tpl_hdr_size, + sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 0) ) AS tpl_data_size, + bool_or(att.atttypid = ''pg_catalog.name''::regtype) + OR sum(CASE WHEN att.attnum > 0 THEN 1 ELSE 0 END) <> count(s.attname) AS is_na + FROM pg_attribute AS att + JOIN pg_class AS tbl ON att.attrelid = tbl.oid + JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace + LEFT JOIN pg_stats AS s ON s.schemaname=ns.nspname + AND s.tablename = tbl.relname AND s.inherited=false AND s.attname=att.attname + LEFT JOIN pg_class AS toast ON tbl.reltoastrelid = toast.oid + WHERE NOT att.attisdropped + AND tbl.relkind in (''r'',''m'') + GROUP BY 1,2,3,4,5,6,7,8,9,10 + ORDER BY 2,3 + ) AS s + ) AS s2 +) AS s3 +-- WHERE NOT is_na +-- AND tblpages*((pst).free_percent + (pst).dead_tuple_percent)::float4/100 >= 1 +ORDER BY schemaname, tblname; + +-- From https://wiki.postgresql.org/wiki/Lock_dependency_information + +CREATE OR REPLACE VIEW blocking_pid_tree AS +WITH RECURSIVE + lock_composite(requested, current) AS (VALUES + (''AccessShareLock''::text, ''AccessExclusiveLock''::text), + (''RowShareLock''::text, ''ExclusiveLock''::text), + (''RowShareLock''::text, ''AccessExclusiveLock''::text), + (''RowExclusiveLock''::text, ''ShareLock''::text), + (''RowExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''RowExclusiveLock''::text, ''ExclusiveLock''::text), + (''RowExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ShareLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''ShareLock''::text, ''RowExclusiveLock''::text), + (''ShareLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ShareLock''::text, ''ShareRowExclusiveLock''::text), + (''ShareLock''::text, ''ExclusiveLock''::text), + (''ShareLock''::text, ''AccessExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''RowExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''ShareLock''::text), + (''ShareRowExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''ExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''ExclusiveLock''::text, ''RowShareLock''::text), + (''ExclusiveLock''::text, ''RowExclusiveLock''::text), + (''ExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ExclusiveLock''::text, ''ShareLock''::text), + (''ExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''ExclusiveLock''::text, ''ExclusiveLock''::text), + (''ExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''AccessShareLock''::text), + (''AccessExclusiveLock''::text, ''RowShareLock''::text), + (''AccessExclusiveLock''::text, ''RowExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''ShareLock''::text), + (''AccessExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''ExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''AccessExclusiveLock''::text) + ) +, lock AS ( + SELECT pid, + virtualtransaction, + granted, + mode, + (locktype, + CASE locktype + WHEN ''relation'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text) + WHEN ''extend'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text) + WHEN ''page'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text, ''page#''||page::text) + WHEN ''tuple'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text, ''page#''||page::text, ''tuple#''||tuple::text) + WHEN ''transactionid'' THEN transactionid::text + WHEN ''virtualxid'' THEN virtualxid::text + WHEN ''object'' THEN concat_ws('';'', ''class:''||classid::regclass::text, ''objid:''||objid, ''col#''||objsubid) + ELSE concat(''db:''||datname) + END::text) AS target + FROM pg_catalog.pg_locks + LEFT JOIN pg_catalog.pg_database ON (pg_database.oid = pg_locks.database) + ) +, waiting_lock AS ( + SELECT + blocker.pid AS blocker_pid, + blocked.pid AS pid, + concat(blocked.mode,blocked.target) AS lock_target + FROM lock blocker + JOIN lock blocked + ON ( NOT blocked.granted + AND blocker.granted + AND blocked.pid != blocker.pid + AND blocked.target IS NOT DISTINCT FROM blocker.target) + JOIN lock_composite c ON (c.requested = blocked.mode AND c.current = blocker.mode) + ) +, acquired_lock AS ( + WITH waiting AS ( + SELECT lock_target, count(lock_target) AS wait_count FROM waiting_lock GROUP BY lock_target + ) + SELECT + pid, + array_agg(concat(mode,target,'' + ''||wait_count) ORDER BY wait_count DESC NULLS LAST) AS locks_acquired + FROM lock + LEFT JOIN waiting ON waiting.lock_target = concat(mode,target) + WHERE granted + GROUP BY pid + ) +, blocking_lock AS ( + SELECT + ARRAY[date_part(''epoch'', query_start)::int, pid] AS seq, + 0::int AS depth, + -1::int AS blocker_pid, + pid, + concat(''Connect: '',usename,'' '',datname,'' '',coalesce(host(client_addr)||'':''||client_port, ''local'') + , E''\nSQL: '',replace(substr(coalesce(query,''N/A''), 1, 60), E''\n'', '' '') + , E''\nAcquired:\n '' + , array_to_string(locks_acquired[1:5] || + CASE WHEN array_upper(locks_acquired,1) > 5 + THEN ''... ''||(array_upper(locks_acquired,1) - 5)::text||'' more ...'' + END, + E''\n '') + ) AS lock_info, + concat(to_char(query_start, CASE WHEN age(query_start) > ''24h'' THEN ''Day DD Mon'' ELSE ''HH24:MI:SS'' END),E'' started\n'' + ,CASE WHEN wait_event IS NOT NULL THEN ''waiting'' ELSE state END,E''\n'' + ,date_trunc(''second'',age(now(),query_start)),'' ago'' + ) AS lock_state + FROM acquired_lock blocker + LEFT JOIN pg_stat_activity act USING (pid) + WHERE EXISTS + (SELECT ''x'' FROM waiting_lock blocked WHERE blocked.blocker_pid = blocker.pid) + AND NOT EXISTS + (SELECT ''x'' FROM waiting_lock blocked WHERE blocked.pid = blocker.pid) +UNION ALL + SELECT + blocker.seq || blocked.pid, + blocker.depth + 1, + blocker.pid, + blocked.pid, + concat(''Connect: '',usename,'' '',datname,'' '',coalesce(host(client_addr)||'':''||client_port, ''local'') + , E''\nSQL: '',replace(substr(coalesce(query,''N/A''), 1, 60), E''\n'', '' '') + , E''\nWaiting: '',blocked.lock_target + , CASE WHEN locks_acquired IS NOT NULL + THEN E''\nAcquired:\n '' || + array_to_string(locks_acquired[1:5] || + CASE WHEN array_upper(locks_acquired,1) > 5 + THEN ''... ''||(array_upper(locks_acquired,1) - 5)::text||'' more ...'' + END, + E''\n '') + END + ) AS lock_info, + concat(to_char(query_start, CASE WHEN age(query_start) > ''24h'' THEN ''Day DD Mon'' ELSE ''HH24:MI:SS'' END),E'' started\n'' + ,CASE WHEN wait_event IS NOT NULL THEN ''waiting'' ELSE state END,E''\n'' + ,date_trunc(''second'',age(now(),query_start)),'' ago'' + ) AS lock_state + FROM blocking_lock blocker + JOIN waiting_lock blocked + ON (blocked.blocker_pid = blocker.pid) + LEFT JOIN pg_stat_activity act ON (act.pid = blocked.pid) + LEFT JOIN acquired_lock acq ON (acq.pid = blocked.pid) + WHERE blocker.depth < 5 + ) +SELECT concat(lpad(''=> '', 4*depth, '' ''),pid::text) AS "PID" +, lock_info AS "Lock Info" +, lock_state AS "State" +FROM blocking_lock +ORDER BY seq; + + +-- From https://wiki.postgresql.org/wiki/Index_Maintenance + +CREATE VIEW duplicate_indexes AS SELECT pg_size_pretty(sum(pg_relation_size(idx))::bigint) as size, + (array_agg(idx))[1] as idx1, (array_agg(idx))[2] as idx2, + (array_agg(idx))[3] as idx3, (array_agg(idx))[4] as idx4 +FROM ( + SELECT indexrelid::regclass as idx, (indrelid::text ||E''\n''|| indclass::text ||E''\n''|| indkey::text ||E''\n''|| + coalesce(indexprs::text,'''')||E''\n'' || coalesce(indpred::text,'''')) as key + FROM pg_index) sub +GROUP BY key HAVING count(*)>1 +ORDER BY sum(pg_relation_size(idx)) DESC; + +-- From https://wiki.postgresql.org/wiki/Disk_Usage + +CREATE VIEW table_sizes AS WITH RECURSIVE pg_inherit(inhrelid, inhparent) AS + (select inhrelid, inhparent + FROM pg_inherits + UNION + SELECT child.inhrelid, parent.inhparent + FROM pg_inherit child, pg_inherits parent + WHERE child.inhparent = parent.inhrelid), +pg_inherit_short AS (SELECT * FROM pg_inherit WHERE inhparent NOT IN (SELECT inhrelid FROM pg_inherit)) +SELECT table_schema + , TABLE_NAME + , row_estimate + , pg_size_pretty(total_bytes) AS total + , pg_size_pretty(index_bytes) AS INDEX + , pg_size_pretty(toast_bytes) AS toast + , pg_size_pretty(table_bytes) AS TABLE + , total_bytes::float8 / sum(total_bytes) OVER () AS total_size_share + FROM ( + SELECT *, total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes + FROM ( + SELECT c.oid + , nspname AS table_schema + , relname AS TABLE_NAME + , SUM(c.reltuples) OVER (partition BY parent) AS row_estimate + , SUM(pg_total_relation_size(c.oid)) OVER (partition BY parent) AS total_bytes + , SUM(pg_indexes_size(c.oid)) OVER (partition BY parent) AS index_bytes + , SUM(pg_total_relation_size(reltoastrelid)) OVER (partition BY parent) AS toast_bytes + , parent + FROM ( + SELECT pg_class.oid + , reltuples + , relname + , relnamespace + , pg_class.reltoastrelid + , COALESCE(inhparent, pg_class.oid) parent + FROM pg_class + LEFT JOIN pg_inherit_short ON inhrelid = oid + WHERE relkind IN (''r'', ''p'') + ) c + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace + ) a + WHERE oid = parent +) a +ORDER BY total_bytes DESC; + +-- From: https://wiki.postgresql.org/wiki/Index_Maintenance + +CREATE VIEW index_usage AS SELECT + t.schemaname, + t.tablename, + c.reltuples::bigint AS num_rows, + pg_size_pretty(pg_relation_size(c.oid)) AS table_size, + psai.indexrelname AS index_name, + pg_size_pretty(pg_relation_size(i.indexrelid)) AS index_size, + CASE WHEN i.indisunique THEN ''Y'' ELSE ''N'' END AS "unique", + psai.idx_scan AS number_of_scans, + psai.idx_tup_read AS tuples_read, + psai.idx_tup_fetch AS tuples_fetched +FROM + pg_tables t + LEFT JOIN pg_class c ON t.tablename = c.relname + LEFT JOIN pg_index i ON c.oid = i.indrelid + LEFT JOIN pg_stat_all_indexes psai ON i.indexrelid = psai.indexrelid +WHERE + t.schemaname NOT IN (''pg_catalog'', ''information_schema'') +ORDER BY 1, 2; + +-- From: https://blog.devgenius.io/top-useful-sql-queries-for-postgresql-35ff3355d265 + +CREATE VIEW database_sizes AS SELECT pg_database.datname, + pg_size_pretty(pg_database_size(pg_database.datname)) AS size +FROM pg_database +ORDER BY pg_database_size(pg_database.datname) DESC; + +CREATE VIEW schema_sizes AS SELECT A.schemaname, + pg_size_pretty (SUM(pg_relation_size(C.oid))) as table, + pg_size_pretty (SUM(pg_total_relation_size(C.oid)-pg_relation_size(C.oid))) as index, + pg_size_pretty (SUM(pg_total_relation_size(C.oid))) as table_index, + SUM(n_live_tup) +FROM pg_class C +LEFT JOIN pg_namespace N ON (N.oid = C .relnamespace) +INNER JOIN pg_stat_user_tables A ON C.relname = A.relname +WHERE nspname NOT IN (''pg_catalog'', ''information_schema'') +AND C .relkind <> ''i'' +AND nspname !~ ''^pg_toast'' +GROUP BY A.schemaname; + +CREATE VIEW last_vacuum_analyze AS SELECT relname, + last_vacuum, + last_autovacuum + n_mod_since_analyze, + last_analyze, + last_autoanalyze, + analyze_count, + autoanalyze_count + FROM pg_stat_user_tables; + +CREATE VIEW table_row_estimates AS SELECT + schemaname, + relname, + n_live_tup +FROM + pg_stat_user_tables +ORDER BY + n_live_tup DESC; + +-- postgres-meta queries as views from: https://github.com/supabase/postgres-meta + +CREATE VIEW pgmeta_columns AS SELECT + c.oid :: int8 AS table_id, + nc.nspname AS schema, + c.relname AS table, + (c.oid || ''.'' || a.attnum) AS id, + a.attnum AS ordinal_position, + a.attname AS name, + CASE + WHEN a.atthasdef THEN pg_get_expr(ad.adbin, ad.adrelid) + ELSE NULL + END AS default_value, + CASE + WHEN t.typtype = ''d'' THEN CASE + WHEN bt.typelem <> 0 :: oid + AND bt.typlen = -1 THEN ''ARRAY'' + WHEN nbt.nspname = ''pg_catalog'' THEN format_type(t.typbasetype, NULL) + ELSE ''USER-DEFINED'' + END + ELSE CASE + WHEN t.typelem <> 0 :: oid + AND t.typlen = -1 THEN ''ARRAY'' + WHEN nt.nspname = ''pg_catalog'' THEN format_type(a.atttypid, NULL) + ELSE ''USER-DEFINED'' + END + END AS data_type, + COALESCE(bt.typname, t.typname) AS format, + a.attidentity IN (''a'', ''d'') AS is_identity, + CASE + a.attidentity + WHEN ''a'' THEN ''ALWAYS'' + WHEN ''d'' THEN ''BY DEFAULT'' + ELSE NULL + END AS identity_generation, + a.attgenerated IN (''s'') AS is_generated, + NOT ( + a.attnotnull + OR t.typtype = ''d'' AND t.typnotnull + ) AS is_nullable, + ( + c.relkind IN (''r'', ''p'') + OR c.relkind IN (''v'', ''f'') AND pg_column_is_updatable(c.oid, a.attnum, FALSE) + ) AS is_updatable, + uniques.table_id IS NOT NULL AS is_unique, + array_to_json( + array( + SELECT + enumlabel + FROM + pg_catalog.pg_enum enums + WHERE + enums.enumtypid = coalesce(bt.oid, t.oid) + OR enums.enumtypid = coalesce(bt.typelem, t.typelem) + ORDER BY + enums.enumsortorder + ) + ) AS enums, + col_description(c.oid, a.attnum) AS comment +FROM + pg_attribute a + LEFT JOIN pg_attrdef ad ON a.attrelid = ad.adrelid + AND a.attnum = ad.adnum + JOIN ( + pg_class c + JOIN pg_namespace nc ON c.relnamespace = nc.oid + ) ON a.attrelid = c.oid + JOIN ( + pg_type t + JOIN pg_namespace nt ON t.typnamespace = nt.oid + ) ON a.atttypid = t.oid + LEFT JOIN ( + pg_type bt + JOIN pg_namespace nbt ON bt.typnamespace = nbt.oid + ) ON t.typtype = ''d'' + AND t.typbasetype = bt.oid + LEFT JOIN ( + SELECT + conrelid AS table_id, + conkey[1] AS ordinal_position + FROM pg_catalog.pg_constraint + WHERE contype = ''u'' AND cardinality(conkey) = 1 + ) AS uniques ON uniques.table_id = c.oid AND uniques.ordinal_position = a.attnum +WHERE + NOT pg_is_other_temp_schema(nc.oid) + AND a.attnum > 0 + AND NOT a.attisdropped + AND (c.relkind IN (''r'', ''v'', ''m'', ''f'', ''p'')) + AND ( + pg_has_role(c.relowner, ''USAGE'') + OR has_column_privilege( + c.oid, + a.attnum, + ''SELECT, INSERT, UPDATE, REFERENCES'' + ) + ); + +CREATE VIEW pgmeta_config AS SELECT + name, + setting, + category, + TRIM(split_part(category, ''/'', 1)) AS group, + TRIM(split_part(category, ''/'', 2)) AS subgroup, + unit, + short_desc, + extra_desc, + context, + vartype, + source, + min_val, + max_val, + enumvals, + boot_val, + reset_val, + sourcefile, + sourceline, + pending_restart +FROM + pg_settings +ORDER BY + category, + name; + +CREATE VIEW pgmeta_extensions AS SELECT + e.name, + n.nspname AS schema, + e.default_version, + x.extversion AS installed_version, + e.comment +FROM + pg_available_extensions() e(name, default_version, comment) + LEFT JOIN pg_extension x ON e.name = x.extname + LEFT JOIN pg_namespace n ON x.extnamespace = n.oid; + +CREATE VIEW pgmeta_foreign_tables AS SELECT + c.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS name, + obj_description(c.oid) AS comment +FROM + pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE + c.relkind = ''f''; + +CREATE VIEW pgmeta_functions AS with functions as ( + select + *, + -- proargmodes is null when all arg modes are IN + coalesce( + p.proargmodes, + array_fill(''i''::text, array[cardinality(coalesce(p.proallargtypes, p.proargtypes))]) + ) as arg_modes, + -- proargnames is null when all args are unnamed + coalesce( + p.proargnames, + array_fill(''''::text, array[cardinality(coalesce(p.proallargtypes, p.proargtypes))]) + ) as arg_names, + -- proallargtypes is null when all arg modes are IN + coalesce(p.proallargtypes, p.proargtypes) as arg_types, + array_cat( + array_fill(false, array[pronargs - pronargdefaults]), + array_fill(true, array[pronargdefaults])) as arg_has_defaults + from + pg_proc as p + where + p.prokind = ''f'' +) +select + f.oid::int8 as id, + n.nspname as schema, + f.proname as name, + l.lanname as language, + case + when l.lanname = ''internal'' then '''' + else f.prosrc + end as definition, + case + when l.lanname = ''internal'' then f.prosrc + else pg_get_functiondef(f.oid) + end as complete_statement, + coalesce(f_args.args, ''[]'') as args, + pg_get_function_arguments(f.oid) as argument_types, + pg_get_function_identity_arguments(f.oid) as identity_argument_types, + f.prorettype::int8 as return_type_id, + pg_get_function_result(f.oid) as return_type, + nullif(rt.typrelid::int8, 0) as return_type_relation_id, + f.proretset as is_set_returning_function, + case + when f.provolatile = ''i'' then ''IMMUTABLE'' + when f.provolatile = ''s'' then ''STABLE'' + when f.provolatile = ''v'' then ''VOLATILE'' + end as behavior, + f.prosecdef as security_definer, + f_config.config_params as config_params +from + functions f + left join pg_namespace n on f.pronamespace = n.oid + left join pg_language l on f.prolang = l.oid + left join pg_type rt on rt.oid = f.prorettype + left join ( + select + oid, + jsonb_object_agg(param, value) filter (where param is not null) as config_params + from + ( + select + oid, + (string_to_array(unnest(proconfig), ''=''))[1] as param, + (string_to_array(unnest(proconfig), ''=''))[2] as value + from + functions + ) as t + group by + oid + ) f_config on f_config.oid = f.oid + left join ( + select + oid, + jsonb_agg(jsonb_build_object( + ''mode'', t2.mode, + ''name'', name, + ''type_id'', type_id, + ''has_default'', has_default + )) as args + from + ( + select + oid, + unnest(arg_modes) as mode, + unnest(arg_names) as name, + unnest(arg_types)::int8 as type_id, + unnest(arg_has_defaults) as has_default + from + functions + ) as t1, + lateral ( + select + case + when t1.mode = ''i'' then ''in'' + when t1.mode = ''o'' then ''out'' + when t1.mode = ''b'' then ''inout'' + when t1.mode = ''v'' then ''variadic'' + else ''table'' + end as mode + ) as t2 + group by + t1.oid + ) f_args on f_args.oid = f.oid; + +CREATE VIEW pgmeta_materialized_views AS select + c.oid::int8 as id, + n.nspname as schema, + c.relname as name, + c.relispopulated as is_populated, + obj_description(c.oid) as comment +from + pg_class c + join pg_namespace n on n.oid = c.relnamespace +where + c.relkind = ''m''; + +CREATE VIEW pgmeta_policies AS SELECT + pol.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS table, + c.oid :: int8 AS table_id, + pol.polname AS name, + CASE + WHEN pol.polpermissive THEN ''PERMISSIVE'' :: text + ELSE ''RESTRICTIVE'' :: text + END AS action, + CASE + WHEN pol.polroles = ''{0}'' :: oid [] THEN array_to_json( + string_to_array(''public'' :: text, '''' :: text) :: name [] + ) + ELSE array_to_json( + ARRAY( + SELECT + pg_roles.rolname + FROM + pg_roles + WHERE + pg_roles.oid = ANY (pol.polroles) + ORDER BY + pg_roles.rolname + ) + ) + END AS roles, + CASE + pol.polcmd + WHEN ''r'' :: "char" THEN ''SELECT'' :: text + WHEN ''a'' :: "char" THEN ''INSERT'' :: text + WHEN ''w'' :: "char" THEN ''UPDATE'' :: text + WHEN ''d'' :: "char" THEN ''DELETE'' :: text + WHEN ''*'' :: "char" THEN ''ALL'' :: text + ELSE NULL :: text + END AS command, + pg_get_expr(pol.polqual, pol.polrelid) AS definition, + pg_get_expr(pol.polwithcheck, pol.polrelid) AS check +FROM + pg_policy pol + JOIN pg_class c ON c.oid = pol.polrelid + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace; + +CREATE VIEW pgmeta_primary_keys AS SELECT + n.nspname AS schema, + c.relname AS table_name, + a.attname AS name, + c.oid :: int8 AS table_id +FROM + pg_index i, + pg_class c, + pg_attribute a, + pg_namespace n +WHERE + i.indrelid = c.oid + AND c.relnamespace = n.oid + AND a.attrelid = c.oid + AND a.attnum = ANY (i.indkey) + AND i.indisprimary; + +CREATE VIEW pgmeta_publications AS SELECT + p.oid :: int8 AS id, + p.pubname AS name, + p.pubowner::regrole::text AS owner, + p.pubinsert AS publish_insert, + p.pubupdate AS publish_update, + p.pubdelete AS publish_delete, + p.pubtruncate AS publish_truncate, + CASE + WHEN p.puballtables THEN NULL + ELSE pr.tables + END AS tables +FROM + pg_catalog.pg_publication AS p + LEFT JOIN LATERAL ( + SELECT + COALESCE( + array_agg( + json_build_object( + ''id'', + c.oid :: int8, + ''name'', + c.relname, + ''schema'', + nc.nspname + ) + ), + ''{}'' + ) AS tables + FROM + pg_catalog.pg_publication_rel AS pr + JOIN pg_class AS c ON pr.prrelid = c.oid + join pg_namespace as nc on c.relnamespace = nc.oid + WHERE + pr.prpubid = p.oid + ) AS pr ON 1 = 1; + +CREATE VIEW pgmeta_relationships AS SELECT + c.oid :: int8 AS id, + c.conname AS constraint_name, + nsa.nspname AS source_schema, + csa.relname AS source_table_name, + sa.attname AS source_column_name, + nta.nspname AS target_table_schema, + cta.relname AS target_table_name, + ta.attname AS target_column_name +FROM + pg_constraint c + JOIN ( + pg_attribute sa + JOIN pg_class csa ON sa.attrelid = csa.oid + JOIN pg_namespace nsa ON csa.relnamespace = nsa.oid + ) ON sa.attrelid = c.conrelid + AND sa.attnum = ANY (c.conkey) + JOIN ( + pg_attribute ta + JOIN pg_class cta ON ta.attrelid = cta.oid + JOIN pg_namespace nta ON cta.relnamespace = nta.oid + ) ON ta.attrelid = c.confrelid + AND ta.attnum = ANY (c.confkey) +WHERE + c.contype = ''f''; + +CREATE VIEW pgmeta_roles AS -- TODO: Consider using pg_authid vs. pg_roles for unencrypted password field +SELECT + oid :: int8 AS id, + rolname AS name, + rolsuper AS is_superuser, + rolcreatedb AS can_create_db, + rolcreaterole AS can_create_role, + rolinherit AS inherit_role, + rolcanlogin AS can_login, + rolreplication AS is_replication_role, + rolbypassrls AS can_bypass_rls, + ( + SELECT + COUNT(*) + FROM + pg_stat_activity + WHERE + pg_roles.rolname = pg_stat_activity.usename + ) AS active_connections, + CASE WHEN rolconnlimit = -1 THEN current_setting(''max_connections'') :: int8 + ELSE rolconnlimit + END AS connection_limit, + rolpassword AS password, + rolvaliduntil AS valid_until, + rolconfig AS config +FROM + pg_roles; + +CREATE VIEW pgmeta_schemas AS select + n.oid::int8 as id, + n.nspname as name, + u.rolname as owner +from + pg_namespace n, + pg_roles u +where + n.nspowner = u.oid + and ( + pg_has_role(n.nspowner, ''USAGE'') + or has_schema_privilege(n.oid, ''CREATE, USAGE'') + ) + and not pg_catalog.starts_with(n.nspname, ''pg_temp_'') + and not pg_catalog.starts_with(n.nspname, ''pg_toast_temp_''); + +CREATE VIEW pgmeta_tables AS SELECT + c.oid :: int8 AS id, + nc.nspname AS schema, + c.relname AS name, + c.relrowsecurity AS rls_enabled, + c.relforcerowsecurity AS rls_forced, + CASE + WHEN c.relreplident = ''d'' THEN ''DEFAULT'' + WHEN c.relreplident = ''i'' THEN ''INDEX'' + WHEN c.relreplident = ''f'' THEN ''FULL'' + ELSE ''NOTHING'' + END AS replica_identity, + pg_total_relation_size(format(''%I.%I'', nc.nspname, c.relname)) :: int8 AS bytes, + pg_size_pretty( + pg_total_relation_size(format(''%I.%I'', nc.nspname, c.relname)) + ) AS size, + pg_stat_get_live_tuples(c.oid) AS live_rows_estimate, + pg_stat_get_dead_tuples(c.oid) AS dead_rows_estimate, + obj_description(c.oid) AS comment +FROM + pg_namespace nc + JOIN pg_class c ON nc.oid = c.relnamespace +WHERE + c.relkind IN (''r'', ''p'') + AND NOT pg_is_other_temp_schema(nc.oid) + AND ( + pg_has_role(c.relowner, ''USAGE'') + OR has_table_privilege( + c.oid, + ''SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER'' + ) + OR has_any_column_privilege(c.oid, ''SELECT, INSERT, UPDATE, REFERENCES'') + ); + + +CREATE VIEW pgmeta_triggers AS SELECT + pg_t.oid AS id, + pg_t.tgrelid AS table_id, + CASE + WHEN pg_t.tgenabled = ''D'' THEN ''DISABLED'' + WHEN pg_t.tgenabled = ''O'' THEN ''ORIGIN'' + WHEN pg_t.tgenabled = ''R'' THEN ''REPLICA'' + WHEN pg_t.tgenabled = ''A'' THEN ''ALWAYS'' + END AS enabled_mode, + ( + STRING_TO_ARRAY( + ENCODE(pg_t.tgargs, ''escape''), ''\000'' + ) + )[:pg_t.tgnargs] AS function_args, + is_t.trigger_name AS name, + is_t.event_object_table AS table, + is_t.event_object_schema AS schema, + is_t.action_condition AS condition, + is_t.action_orientation AS orientation, + is_t.action_timing AS activation, + ARRAY_AGG(is_t.event_manipulation)::text[] AS events, + pg_p.proname AS function_name, + pg_n.nspname AS function_schema +FROM + pg_trigger AS pg_t +JOIN + pg_class AS pg_c +ON pg_t.tgrelid = pg_c.oid +JOIN information_schema.triggers AS is_t +ON is_t.trigger_name = pg_t.tgname +AND pg_c.relname = is_t.event_object_table +JOIN pg_proc AS pg_p +ON pg_t.tgfoid = pg_p.oid +JOIN pg_namespace AS pg_n +ON pg_p.pronamespace = pg_n.oid +GROUP BY + pg_t.oid, + pg_t.tgrelid, + pg_t.tgenabled, + pg_t.tgargs, + pg_t.tgnargs, + is_t.trigger_name, + is_t.event_object_table, + is_t.event_object_schema, + is_t.action_condition, + is_t.action_orientation, + is_t.action_timing, + pg_p.proname, + pg_n.nspname; + + +CREATE VIEW pgmeta_types AS select + t.oid::int8 as id, + t.typname as name, + n.nspname as schema, + format_type (t.oid, null) as format, + coalesce(t_enums.enums, ''[]'') as enums, + coalesce(t_attributes.attributes, ''[]'') as attributes, + obj_description (t.oid, ''pg_type'') as comment +from + pg_type t + left join pg_namespace n on n.oid = t.typnamespace + left join ( + select + enumtypid, + jsonb_agg(enumlabel order by enumsortorder) as enums + from + pg_enum + group by + enumtypid + ) as t_enums on t_enums.enumtypid = t.oid + left join ( + select + oid, + jsonb_agg( + jsonb_build_object(''name'', a.attname, ''type_id'', a.atttypid::int8) + order by a.attnum asc + ) as attributes + from + pg_class c + join pg_attribute a on a.attrelid = c.oid + where + c.relkind = ''c'' and not a.attisdropped + group by + c.oid + ) as t_attributes on t_attributes.oid = t.typrelid +where + ( + t.typrelid = 0 + or ( + select + c.relkind = ''c'' + from + pg_class c + where + c.oid = t.typrelid + ) + ); + +CREATE VIEW pgmeta_version AS SELECT + version(), + current_setting(''server_version_num'') :: int8 AS version_number, + ( + SELECT + COUNT(*) AS active_connections + FROM + pg_stat_activity + ) AS active_connections, + current_setting(''max_connections'') :: int8 AS max_connections; + +CREATE VIEW pgmeta_views AS SELECT + c.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS name, + -- See definition of information_schema.views + (pg_relation_is_updatable(c.oid, false) & 20) = 20 AS is_updatable, + obj_description(c.oid) AS comment +FROM + pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE + c.relkind = ''v''; +', + ' +# michelp-adminpack + +A Trusted Language Extension containing a variety of useful databse admin queries. + +Postgres database admins are often faced with a variety of issues that +require them introspecting the database state. This extension +contains a mixed-bag of views that reflect useful information for +admins. + +## Installation + +Using dbdev: + +``` +postgres=> select dbdev.install(''michelp-adminpack''); + install +--------- + t +(1 row) + +postgres=> create schema adminpack; +CREATE SCHEMA +postgres=> create extension "michelp-adminpack" with schema adminpack; +CREATE EXTENSION +``` + +This will the extension into the `adminpack` schema, substitute with +some other schema if you want it to go somewhere else. + +## Index Bloat + +Index updates and querying can end up getting slower and slower over +time due to what''s called "index bloat". This is where frequent +updates and deletes can cause space in index data structures to go +unused. Understanding when this is happening is not obvious, so there +is a rather complex query you can run to detect it. + +`index_bloat` + +| Column | Type | +|------------------|------------------| +| current_database | name | +| schemaname | name | +| tblname | name | +| idxname | name | +| real_size | numeric | +| extra_size | numeric | +| extra_pct | double precision | +| fillfactor | integer | +| bloat_size | double precision | +| bloat_pct | double precision | +| is_na | boolean | + + +## Table Bloat + +Like index bloat, tables with high update and delete rates can also +end up containing lots of allocated but unused space. This query +shows which tables have the most bloat. + +`table_bloat` + +| Column | Type | +|------------------|------------------| +| current_database | name | +| schemaname | name | +| tblname | name | +| real_size | numeric | +| extra_size | double precision | +| extra_pct | double precision | +| fillfactor | integer | +| bloat_size | double precision | +| bloat_pct | double precision | +| is_na | boolean | + + +## Blocking PID Tree + +Postgres queries can block each other, and this blocking relationship +can form a tree, where A blocks B, which blocks C, and so on. This +query formats blocking queries into a tree structure so it''s easy to +see what query is causing the blockage. + +`blocking_pid_tree` + +| Column | Type | +|-----------|------| +| PID | text | +| Lock Info | text | +| State | text | + + +## Duplicate Indexes + +If a table contains duplicate indexes, then unecessary work is done +updating and storing them, this query will show up to 4 duplicate +indexes per table if they exist. + +`duplicate_indexes` + +| Column | Type | +|--------|----------| +| size | text | +| idx1 | regclass | +| idx2 | regclass | +| idx3 | regclass | +| idx4 | regclass | + + +## Table Sizes + +This view shows tables and their sizes. + +`table_sizes` + +| Column | Type | +|------------------|------------------| +| table_schema | name | +| table_name | name | +| row_estimate | real | +| total | text | +| index | text | +| toast | text | +| table | text | +| total_size_share | double precision | + + +## Schema Sizes + +This view shows schemas and their sizes, which is the sum of the sizes +of all the tables and indexes in the schema. + +`schema_sizes` + +| Column | Type | +|-------------|---------| +| schemaname | name | +| table | text | +| index | text | +| table_index | text | +| sum | numeric | + + +## Index Usage + +This view shows index size and usage. An unused index still needs to +be updated and that takes time an storage, so it''s a good candidate to +drop. + +`index_usage` + +| Column | Type | +|-----------------|--------| +| schemaname | name | +| tablename | name | +| num_rows | bigint | +| table_size | text | +| index_name | name | +| index_size | text | +| unique | text | +| number_of_scans | bigint | +| tuples_read | bigint | +| tuples_fetched | bigint | + + +## Last Vacuum Analyze + +This views shows the last time a table was vacuumed an analyzed. + +`last_vacuum_analyze` + +| Column | Type | +|---------------------|--------------------------| +| relname | name | +| last_vacuum | timestamp with time zone | +| n_mod_since_analyze | timestamp with time zone | +| last_analyze | timestamp with time zone | +| last_autoanalyze | timestamp with time zone | +| analyze_count | bigint | +| autoanalyze_count | bigint | + +## Table Row Estimates + +This view shows estimates for the number of rows in a table. This is +just an estimate and depends on up to date table statistics. + +`table_row_estimates` + +| Column | Type | +|------------|--------| +| schemaname | name | +| relname | name | +| n_live_tup | bigint | + +## PGMeta Columns + +This view shows infromation for the columns of tables in the system. + +`pgmeta_columns` + +| Column | Type | +|---------------------|----------| +| table_id | bigint | +| schema | name | +| table | name | +| id | text | +| ordinal_position | smallint | +| name | name | +| default_value | text | +| data_type | text | +| format | name | +| is_identity | boolean | +| identity_generation | text | +| is_generated | boolean | +| is_nullable | boolean | +| is_updatable | boolean | +| is_unique | boolean | +| enums | json | +| comment | text | + +## PGMeta Config + +This views shows the configuration of the database. + +`pgmeta_config` + +| Column | Type | +|-----------------|---------| +| name | text | +| setting | text | +| category | text | +| group | text | +| subgroup | text | +| unit | text | +| short_desc | text | +| extra_desc | text | +| context | text | +| vartype | text | +| source | text | +| min_val | text | +| max_val | text | +| enumvals | text[] | +| boot_val | text | +| reset_val | text | +| sourcefile | text | +| sourceline | integer | +| pending_restart | boolean | + +## PGMeta Extensions + +This view shows installed extensions in the database. + +`pgmeta_extensions` + +| Column | Type | +|-------------------|------| +| name | name | +| schema | name | +| default_version | text | +| installed_version | text | +| comment | text | + +## PGMeta Foreign Tables + +This view shows foreign tables in the database. + +`pgmeta_foreign_tables` + +| Column | Type | +|---------|--------| +| id | bigint | +| schema | name | +| name | name | +| comment | text | + +## PGMeta Functions + +This view shows functions in the database. + +`pgmeta_functions` + +| Column | Type | +|---------------------------|---------| +| id | bigint | +| schema | name | +| name | name | +| language | name | +| definition | text | +| complete_statement | text | +| args | jsonb | +| argument_types | text | +| identity_argument_types | text | +| return_type_id | bigint | +| return_type | text | +| return_type_relation_id | bigint | +| is_set_returning_function | boolean | +| behavior | text | +| security_definer | boolean | +| config_params | jsonb | + +## PGMeta Materialized Views + +This view shows materialized views in the database. + +`pgmeta_materialized_views` + +| Column | Type | +|--------------|---------| +| id | bigint | +| schema | name | +| name | name | +| is_populated | boolean | +| comment | text | + +## PGMeta Policies + +This view shows Row Level Security Policies in the database. + +`pgmeta_policies` + +| Column | Type | +|------------|--------| +| id | bigint | +| schema | name | +| table | name | +| table_id | bigint | +| name | name | +| action | text | +| roles | json | +| command | text | +| definition | text | +| check | text | + +## PGMeta Primary Keys + +This view shows primary keys in the database. + +`pgmeta_primary_keys` + +| Column | Type | +|------------|--------| +| schema | name | +| table_name | name | +| name | name | +| table_id | bigint | + +## PGMeta Publications + +This view shows logical replication publishers in the database. + +`pgmeta_publications` + +| Column | Type | +|------------------|---------| +| id | bigint | +| name | name | +| owner | text | +| publish_insert | boolean | +| publish_update | boolean | +| publish_delete | boolean | +| publish_truncate | boolean | +| tables | json[] | + +## PGMeta Relationships + +This view shows foreign key relationships in the database. + +`pgmeta_relationships` + +| Column | Type | +|---------------------|--------| +| id | bigint | +| constraint_name | name | +| source_schema | name | +| source_table_name | name | +| source_column_name | name | +| target_table_schema | name | +| target_table_name | name | +| target_column_name | name | + +## PGMeta Roles + +This view shows roles in the database system. Note that roles are +global objects and apply to all databases. + +`pgmeta_roles` + +| Column | Type | +|---------------------|--------------------------| +| id | bigint | +| name | name | +| is_superuser | boolean | +| can_create_db | boolean | +| can_create_role | boolean | +| inherit_role | boolean | +| can_login | boolean | +| is_replication_role | boolean | +| can_bypass_rls | boolean | +| active_connections | bigint | +| connection_limit | bigint | +| password | text | +| valid_until | timestamp with time zone | +| config | text[] | + +## PGMeta Schemas + +This view shows all schemas in the database. + +`pgmeta_schemas` + +| Column | Type | +|--------|--------| +| id | bigint | +| name | name | +| owner | name | + +## PGMeta Tables + +This view shows all tables in the database. + +`pgmeta_tables` + +| Column | Type | +|--------------------|---------| +| id | bigint | +| schema | name | +| name | name | +| rls_enabled | boolean | +| rls_forced | boolean | +| replica_identity | text | +| bytes | bigint | +| size | text | +| live_rows_estimate | bigint | +| dead_rows_estimate | bigint | +| comment | text | + +## PGMeta Triggers + +This view shows all triggers in the database. + +`pgmeta_triggers` + +| Column | Type | +|-----------------|-----------------------------------| +| id | oid | +| table_id | oid | +| enabled_mode | text | +| function_args | text[] | +| name | information_schema.sql_identifier | +| table | information_schema.sql_identifier | +| schema | information_schema.sql_identifier | +| condition | information_schema.character_data | +| orientation | information_schema.character_data | +| activation | information_schema.character_data | +| events | text[] | +| function_name | name | +| function_schema | name | + +## PGMeta Types + +This view shows all types in the database. + +`pgmeta_types` + +| Column | Type | +|------------|--------| +| id | bigint | +| name | name | +| schema | name | +| format | text | +| enums | jsonb | +| attributes | jsonb | +| comment | text | + +## PGMeta Version + +This view shows the current database version. + +`pgmeta_version` + +| Column | Type | +|--------------------|--------| +| version | text | +| version_number | bigint | +| active_connections | bigint | +| max_connections | bigint | + +## PGMeta Views + +This view shows all views in the database. + +`pgmeta_views` + +| Column | Type | +|--------------|---------| +| id | bigint | +| schema | name | +| name | name | +| is_updatable | boolean | +| comment | text | +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405083103_fix_auth_schema_values_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405083103_fix_auth_schema_values_100.snap new file mode 100644 index 000000000..98e95001f --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405083103_fix_auth_schema_values_100.snap @@ -0,0 +1,33 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230405083103_fix_auth_schema_values.sql +--- +update auth.users +set created_at = NOW(), +updated_at = NOW(), +email_confirmed_at = NOW(), +confirmation_token = '', +recovery_token = '', +email_change_token_new = '', +email_change = ''; + +insert into auth.identities ( + id, + provider_id, + provider, + user_id, + identity_data, + created_at, + updated_at +) +select + gen_random_uuid(), + email, + 'email' as provider, + id as user_id, + jsonb_build_object('sub', id, 'email', email) + as identity_data, + created_at, + updated_at +from + auth.users; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405083103_fix_auth_schema_values_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405083103_fix_auth_schema_values_80.snap new file mode 100644 index 000000000..629156e54 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405083103_fix_auth_schema_values_80.snap @@ -0,0 +1,38 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230405083103_fix_auth_schema_values.sql +--- +update auth.users +set created_at = NOW(), +updated_at = NOW(), +email_confirmed_at = NOW(), +confirmation_token = '', +recovery_token = '', +email_change_token_new = '', +email_change = ''; + +insert into auth.identities ( + id, + provider_id, + provider, + user_id, + identity_data, + created_at, + updated_at +) +select + gen_random_uuid(), + email, + 'email' as provider, + id as user_id, + jsonb_build_object( + 'sub', + id, + 'email', + email + ) + as identity_data, + created_at, + updated_at +from + auth.users; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405085810_fix_avatars_handle_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405085810_fix_avatars_handle_100.snap new file mode 100644 index 000000000..fc7d786d4 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405085810_fix_avatars_handle_100.snap @@ -0,0 +1,44 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230405085810_fix_avatars_handle.sql +--- +create or replace function app.update_avatar_id() +returns trigger +language plpgsql +security definer +as $function$ +declare + v_handle app.valid_name; + v_affected_account app.accounts := null; + begin + select (string_to_array(new.name, '/'::text))[1]::app.valid_name into v_handle; + + update app.accounts + set avatar_id = new.id + where handle = v_handle + returning * into v_affected_account; + + if not v_affected_account is null then + update auth.users u + set + "raw_user_meta_data" = u.raw_user_meta_data || jsonb_build_object( + 'avatar_path', new.name + ) + where u.id = v_affected_account.id; + else + update app.organizations + set avatar_id = new.id + where handle = v_handle; + end if; + + return new; + end; +$function$; + +alter policy storage_objects_insert_policy +on storage.objects +with check (app.is_handle_maintainer( + auth.uid(), + cast((string_to_array(name, cast('/' as text)))[1] + as app.valid_name) +)); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405085810_fix_avatars_handle_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405085810_fix_avatars_handle_80.snap new file mode 100644 index 000000000..fc7d786d4 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405085810_fix_avatars_handle_80.snap @@ -0,0 +1,44 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230405085810_fix_avatars_handle.sql +--- +create or replace function app.update_avatar_id() +returns trigger +language plpgsql +security definer +as $function$ +declare + v_handle app.valid_name; + v_affected_account app.accounts := null; + begin + select (string_to_array(new.name, '/'::text))[1]::app.valid_name into v_handle; + + update app.accounts + set avatar_id = new.id + where handle = v_handle + returning * into v_affected_account; + + if not v_affected_account is null then + update auth.users u + set + "raw_user_meta_data" = u.raw_user_meta_data || jsonb_build_object( + 'avatar_path', new.name + ) + where u.id = v_affected_account.id; + else + update app.organizations + set avatar_id = new.id + where handle = v_handle; + end if; + + return new; + end; +$function$; + +alter policy storage_objects_insert_policy +on storage.objects +with check (app.is_handle_maintainer( + auth.uid(), + cast((string_to_array(name, cast('/' as text)))[1] + as app.valid_name) +)); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405163940_download_metrics_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405163940_download_metrics_100.snap new file mode 100644 index 000000000..25d67fb34 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405163940_download_metrics_100.snap @@ -0,0 +1,102 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230405163940_download_metrics.sql +--- +create or replace function app.api_request_headers() +returns json +language sql +stable +as $function$ +select coalesce(current_setting('request.headers', true)::json, '{}'::json); +$function$; + +create or replace function app.api_request_header(text) +returns text +language sql +stable +as $function$ +select app.api_request_headers() ->> $1 +$function$; + +create or replace function app.api_request_ip() +returns inet +language sql +stable +as $function$ +select split_part(app.api_request_header('x-forwarded-for') || ',', ',', 1)::inet +$function$; + +create or replace function app.api_request_client_info() +returns text +language sql +stable +as $function$ +select app.api_request_header('x-client-info') +$function$; + +create table app.downloads ( + id uuid primary key default gen_random_uuid(), + package_id uuid + not null + references app.packages (id), + ip inet + not null + default cast(app.api_request_ip() as inet), + client_info text + default app.api_request_client_info(), + created_at timestamp with time zone + not null + default NOW() +); + +create index "downloads_package_id_ip" on app.downloads using btree (package_id); + +create index "downloads_created_at" on app.downloads using brin (created_at); + +create or replace function public.register_download(package_name text) +returns void +language sql +security definer +as $function$ +insert into app.downloads(package_id) + select id + from app.packages ap + where ap.package_name = $1 +$function$; + +create materialized view public.download_metrics +as + select + dl.package_id, + COUNT(dl.id) as downloads_all_time, + COUNT( + dl.id + ) + filter (where + dl.created_at > + NOW() - cast('180 days' as interval)) + as downloads_180_days, + COUNT( + dl.id + ) + filter (where + dl.created_at > + NOW() - cast('90 days' as interval)) + as downloads_90_days, + COUNT( + dl.id + ) + filter (where + dl.created_at > + NOW() - cast('30 days' as interval)) + as downloads_30_day + from + app.downloads as dl + group by dl.package_id; + +select + cron.schedule( + 'refresh download metrics', + '*/30 * * * *', + 'refresh materialized view public.download_metrics;' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405163940_download_metrics_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405163940_download_metrics_80.snap new file mode 100644 index 000000000..339ac6435 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230405163940_download_metrics_80.snap @@ -0,0 +1,109 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230405163940_download_metrics.sql +--- +create or replace function app.api_request_headers() +returns json +language sql +stable +as $function$ +select coalesce(current_setting('request.headers', true)::json, '{}'::json); +$function$; + +create or replace function app.api_request_header(text) +returns text +language sql +stable +as $function$ +select app.api_request_headers() ->> $1 +$function$; + +create or replace function app.api_request_ip() +returns inet +language sql +stable +as $function$ +select split_part(app.api_request_header('x-forwarded-for') || ',', ',', 1)::inet +$function$; + +create or replace function app.api_request_client_info() +returns text +language sql +stable +as $function$ +select app.api_request_header('x-client-info') +$function$; + +create table app.downloads ( + id uuid + primary key + default gen_random_uuid(), + package_id uuid + not null + references app.packages (id), + ip inet + not null + default cast(app.api_request_ip() as inet), + client_info text + default app.api_request_client_info(), + created_at timestamp with time zone + not null + default NOW() +); + +create index "downloads_package_id_ip" +on app.downloads +using btree +( + package_id +); + +create index "downloads_created_at" on app.downloads using brin (created_at); + +create or replace function public.register_download(package_name text) +returns void +language sql +security definer +as $function$ +insert into app.downloads(package_id) + select id + from app.packages ap + where ap.package_name = $1 +$function$; + +create materialized view public.download_metrics +as + select + dl.package_id, + COUNT(dl.id) as downloads_all_time, + COUNT( + dl.id + ) + filter (where + dl.created_at > + NOW() - cast('180 days' as interval)) + as downloads_180_days, + COUNT( + dl.id + ) + filter (where + dl.created_at > + NOW() - cast('90 days' as interval)) + as downloads_90_days, + COUNT( + dl.id + ) + filter (where + dl.created_at > + NOW() - cast('30 days' as interval)) + as downloads_30_day + from + app.downloads as dl + group by dl.package_id; + +select + cron.schedule( + 'refresh download metrics', + '*/30 * * * *', + 'refresh materialized view public.download_metrics;' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411104448_download_metrics_computed_relation_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411104448_download_metrics_computed_relation_100.snap new file mode 100644 index 000000000..2d6a2884b --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411104448_download_metrics_computed_relation_100.snap @@ -0,0 +1,14 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230411104448_download_metrics_computed_relation.sql +--- +create or replace function public.download_metrics(public.packages) +returns setof public.download_metrics +language sql +stable +rows 1 +as $function$ +select * + from public.download_metrics dm + where dm.package_id = $1.id; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411104448_download_metrics_computed_relation_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411104448_download_metrics_computed_relation_80.snap new file mode 100644 index 000000000..2d6a2884b --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411104448_download_metrics_computed_relation_80.snap @@ -0,0 +1,14 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230411104448_download_metrics_computed_relation.sql +--- +create or replace function public.download_metrics(public.packages) +returns setof public.download_metrics +language sql +stable +rows 1 +as $function$ +select * + from public.download_metrics dm + where dm.package_id = $1.id; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175952_langchain-embedding_search_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175952_langchain-embedding_search_100.snap new file mode 100644 index 000000000..a38f3e7d9 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175952_langchain-embedding_search_100.snap @@ -0,0 +1,148 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230411175952_langchain-embedding_search.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'langchain', + 'embedding_search', + 'Search document embeddings', + true, + '{pg_vector}' + ); + +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_name = 'langchain-embedding_search' + ), + (1, 0, 0), + ' +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''vector'' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception ''"langchain-embedding_search" requires "vector"'' + using hint = ''Run "create extension vector" and try again''; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +', + ' +# embedding_search + +[LangChain](https://js.langchain.com/docs/) is a framework for developing applications powered by language models with a plugable architecture. + +`langchain-embedding_search` uses a Supabase Postgres database as its vector store. + +## Installation + +```sql +select dbdev.install(''langchain-embedding_search''); +create extension if not exists vector; +create extension "langchain-embedding_search" + schema public + version ''1.0.0''; +``` +Note: + +`vector` is a dependency of `langchain-embedding_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromTexts( + ["Hello world", "Bye bye", "What''s this?"], + [{ id: 2 }, { id: 1 }, { id: 3 }], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + const resultOne = await vectorStore.similaritySearch("Hello world", 1); + + console.log(resultOne); +}; +``` + +For more details, checkout the LangChain Supabase integration docs: https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175952_langchain-embedding_search_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175952_langchain-embedding_search_80.snap new file mode 100644 index 000000000..ee55c63f7 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175952_langchain-embedding_search_80.snap @@ -0,0 +1,154 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230411175952_langchain-embedding_search.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'langchain', + 'embedding_search', + 'Search document embeddings', + true, + '{pg_vector}' + ); + +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_name = + 'langchain-embedding_search' + ), + (1, 0, 0), + ' +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''vector'' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception ''"langchain-embedding_search" requires "vector"'' + using hint = ''Run "create extension vector" and try again''; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +', + ' +# embedding_search + +[LangChain](https://js.langchain.com/docs/) is a framework for developing applications powered by language models with a plugable architecture. + +`langchain-embedding_search` uses a Supabase Postgres database as its vector store. + +## Installation + +```sql +select dbdev.install(''langchain-embedding_search''); +create extension if not exists vector; +create extension "langchain-embedding_search" + schema public + version ''1.0.0''; +``` +Note: + +`vector` is a dependency of `langchain-embedding_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromTexts( + ["Hello world", "Bye bye", "What''s this?"], + [{ id: 2 }, { id: 1 }, { id: 3 }], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + const resultOne = await vectorStore.similaritySearch("Hello world", 1); + + console.log(resultOne); +}; +``` + +For more details, checkout the LangChain Supabase integration docs: https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175953_langchain-hybrid_search_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175953_langchain-hybrid_search_100.snap new file mode 100644 index 000000000..5bd9c4cd2 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175953_langchain-hybrid_search_100.snap @@ -0,0 +1,169 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230411175953_langchain-hybrid_search.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'langchain', + 'hybrid_search', + 'Search documents by embedding and full text', + true, + '{pg_vector}' + ); + +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_name = 'langchain-hybrid_search' + ), + (1, 0, 0), + ' +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''vector'' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception ''"langchain-hybrid_search" requires "vector"'' + using hint = ''Run "create extension vector" and try again''; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to similarity search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +-- Create a function to keyword search for documents +create function kw_match_documents(query_text text, match_count int) +returns table (id bigint, content text, metadata jsonb, similarity real) +as $$ + +begin +return query execute +format('' + select + id, content, metadata, ts_rank(to_tsvector(content), plainto_tsquery($1)) as similarity + from + documents + where + to_tsvector(content) @@ plainto_tsquery($1) + order by + similarity desc + limit $2 +'') +using query_text, match_count; +end; +$$ language plpgsql; + +', + ' +# hybrid_search + +Langchain supports hybrid search with a Supabase Postgres database. The hybrid search combines the postgres pgvector extension (similarity search) and Full-Text Search (keyword search) to retrieve documents. You can add documents via SupabaseVectorStore addDocuments function. SupabaseHybridKeyWordSearch accepts embedding, supabase client, number of results for similarity search, and number of results for keyword search as parameters. The getRelevantDocuments function produces a list of documents that has duplicates removed and is sorted by relevance score. + +## Installation + +```sql +select dbdev.install(''langchain-hybrid_search''); +create extension if not exists vector; +create extension "langchain-hybrid_search" + schema public + version ''1.0.0''; +``` +Note: + +`vector` is a dependency of `langchain-hybrid_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; +import { SupabaseHybridSearch } from "langchain/retrievers/supabase"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const embeddings = new OpenAIEmbeddings(); + + const retriever = new SupabaseHybridSearch(embeddings, { + client, + // Below are the defaults, expecting that you set up your supabase table and functions according to the guide above. Please change if necessary. + similarityK: 2, + keywordK: 2, + tableName: "documents", + similarityQueryName: "match_documents", + keywordQueryName: "kw_match_documents", + }); + + const results = await retriever.getRelevantDocuments("hello bye"); + + console.log(results); +}; +``` + +For more details, checkout the LangChain Supabase Hybrid Search docs: https://js.langchain.com/docs/modules/indexes/retrievers/supabase-hybrid +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175953_langchain-hybrid_search_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175953_langchain-hybrid_search_80.snap new file mode 100644 index 000000000..e822ff902 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230411175953_langchain-hybrid_search_80.snap @@ -0,0 +1,174 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230411175953_langchain-hybrid_search.sql +--- +insert into app.packages ( + handle, + partial_name, + control_description, + control_relocatable, + control_requires +) +values + ( + 'langchain', + 'hybrid_search', + 'Search documents by embedding and full text', + true, + '{pg_vector}' + ); + +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_name = 'langchain-hybrid_search' + ), + (1, 0, 0), + ' +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''vector'' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception ''"langchain-hybrid_search" requires "vector"'' + using hint = ''Run "create extension vector" and try again''; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to similarity search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +-- Create a function to keyword search for documents +create function kw_match_documents(query_text text, match_count int) +returns table (id bigint, content text, metadata jsonb, similarity real) +as $$ + +begin +return query execute +format('' + select + id, content, metadata, ts_rank(to_tsvector(content), plainto_tsquery($1)) as similarity + from + documents + where + to_tsvector(content) @@ plainto_tsquery($1) + order by + similarity desc + limit $2 +'') +using query_text, match_count; +end; +$$ language plpgsql; + +', + ' +# hybrid_search + +Langchain supports hybrid search with a Supabase Postgres database. The hybrid search combines the postgres pgvector extension (similarity search) and Full-Text Search (keyword search) to retrieve documents. You can add documents via SupabaseVectorStore addDocuments function. SupabaseHybridKeyWordSearch accepts embedding, supabase client, number of results for similarity search, and number of results for keyword search as parameters. The getRelevantDocuments function produces a list of documents that has duplicates removed and is sorted by relevance score. + +## Installation + +```sql +select dbdev.install(''langchain-hybrid_search''); +create extension if not exists vector; +create extension "langchain-hybrid_search" + schema public + version ''1.0.0''; +``` +Note: + +`vector` is a dependency of `langchain-hybrid_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; +import { SupabaseHybridSearch } from "langchain/retrievers/supabase"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const embeddings = new OpenAIEmbeddings(); + + const retriever = new SupabaseHybridSearch(embeddings, { + client, + // Below are the defaults, expecting that you set up your supabase table and functions according to the guide above. Please change if necessary. + similarityK: 2, + keywordK: 2, + tableName: "documents", + similarityQueryName: "match_documents", + keywordQueryName: "kw_match_documents", + }); + + const results = await retriever.getRelevantDocuments("hello bye"); + + console.log(results); +}; +``` + +For more details, checkout the LangChain Supabase Hybrid Search docs: https://js.langchain.com/docs/modules/indexes/retrievers/supabase-hybrid +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413130634_popular_packages_function_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413130634_popular_packages_function_100.snap new file mode 100644 index 000000000..c903d23c6 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413130634_popular_packages_function_100.snap @@ -0,0 +1,16 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230413130634_popular_packages_function.sql +--- +create or replace function public.popular_packages() +returns setof public.packages +language sql +stable +as $function$ +select * from public.packages p + order by ( + select (dm.downloads_30_day * 5) + (dm.downloads_90_days * 2) + dm.downloads_180_days + from public.download_metrics dm + where dm.package_id = p.id + ) desc nulls last, p.created_at desc; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413130634_popular_packages_function_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413130634_popular_packages_function_80.snap new file mode 100644 index 000000000..c903d23c6 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413130634_popular_packages_function_80.snap @@ -0,0 +1,16 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230413130634_popular_packages_function.sql +--- +create or replace function public.popular_packages() +returns setof public.packages +language sql +stable +as $function$ +select * from public.packages p + order by ( + select (dm.downloads_30_day * 5) + (dm.downloads_90_days * 2) + dm.downloads_180_days + from public.download_metrics dm + where dm.package_id = p.id + ) desc nulls last, p.created_at desc; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413140356_update_profile_function_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413140356_update_profile_function_100.snap new file mode 100644 index 000000000..71c869748 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413140356_update_profile_function_100.snap @@ -0,0 +1,27 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230413140356_update_profile_function.sql +--- +create or replace function +public.update_profile( + handle app.valid_name, + display_name text default null, + bio text default null +) +returns void +language plpgsql +as $function$ +declare + v_is_org boolean; + begin + update app.accounts a + set display_name = coalesce($2, a.display_name), + bio = coalesce($3, a.bio) + where a.handle = $1; + + update app.organizations o + set display_name = coalesce($2, o.display_name), + bio = coalesce($3, o.bio) + where o.handle = $1; + end; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413140356_update_profile_function_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413140356_update_profile_function_80.snap new file mode 100644 index 000000000..71c869748 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230413140356_update_profile_function_80.snap @@ -0,0 +1,27 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230413140356_update_profile_function.sql +--- +create or replace function +public.update_profile( + handle app.valid_name, + display_name text default null, + bio text default null +) +returns void +language plpgsql +as $function$ +declare + v_is_org boolean; + begin + update app.accounts a + set display_name = coalesce($2, a.display_name), + bio = coalesce($3, a.bio) + where a.handle = $1; + + update app.organizations o + set display_name = coalesce($2, o.display_name), + bio = coalesce($3, o.bio) + where o.handle = $1; + end; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230417141004_dbdev_short_desc_typo_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230417141004_dbdev_short_desc_typo_100.snap new file mode 100644 index 000000000..c0879f908 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230417141004_dbdev_short_desc_typo_100.snap @@ -0,0 +1,8 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230417141004_dbdev_short_desc_typo.sql +--- +update app.packages +set control_description = 'Install packages from the database.dev registry' +where + handle = 'supabase' and partial_name = 'dbdev'; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230417141004_dbdev_short_desc_typo_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230417141004_dbdev_short_desc_typo_80.snap new file mode 100644 index 000000000..b55a9e58c --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230417141004_dbdev_short_desc_typo_80.snap @@ -0,0 +1,9 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230417141004_dbdev_short_desc_typo.sql +--- +update app.packages +set control_description = 'Install packages from the database.dev registry' +where + handle = 'supabase' and + partial_name = 'dbdev'; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508165641_packages_order_version_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508165641_packages_order_version_100.snap new file mode 100644 index 000000000..68d356d7d --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508165641_packages_order_version_100.snap @@ -0,0 +1,28 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230508165641_packages_order_version.sql +--- +create or replace view public.packages +as select + pa.id, + pa.package_name, + pa.handle, + pa.partial_name, + newest_ver.version as latest_version, + newest_ver.description_md, + pa.control_description, + pa.control_requires, + pa.created_at +from + app.packages as pa, + lateral ( + select + * + from + app.package_versions as pv + where + pv.package_id = pa.id + order by pv.version_struct desc + limit 1 + ) + as newest_ver; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508165641_packages_order_version_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508165641_packages_order_version_80.snap new file mode 100644 index 000000000..68d356d7d --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508165641_packages_order_version_80.snap @@ -0,0 +1,28 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230508165641_packages_order_version.sql +--- +create or replace view public.packages +as select + pa.id, + pa.package_name, + pa.handle, + pa.partial_name, + newest_ver.version as latest_version, + newest_ver.description_md, + pa.control_description, + pa.control_requires, + pa.created_at +from + app.packages as pa, + lateral ( + select + * + from + app.package_versions as pv + where + pv.package_id = pa.id + order by pv.version_struct desc + limit 1 + ) + as newest_ver; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175952_langchain-embedding_search-1_1_0_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175952_langchain-embedding_search-1_1_0_100.snap new file mode 100644 index 000000000..c18b7bb35 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175952_langchain-embedding_search-1_1_0_100.snap @@ -0,0 +1,178 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230508175952_langchain-embedding_search-1_1_0.sql +--- +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_name = 'langchain-embedding_search' + ), + (1, 1, 0), + ' +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''vector'' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception ''"langchain-embedding_search" requires "vector"'' + using hint = ''Run "create extension vector" and try again''; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int, + filter jsonb DEFAULT ''{}'' +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + where metadata @> filter + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +', + ' +# embedding_search + +[LangChain](https://js.langchain.com/docs/) is a framework for developing applications powered by language models with a plugable architecture. + +`langchain-embedding_search` uses a Supabase Postgres database as its vector store. + +## Installation + +```sql +select dbdev.install(''langchain-embedding_search''); +create extension if not exists vector; +create extension "langchain-embedding_search" + schema public + version ''1.1.0''; +``` +Note: + +`vector` is a dependency of `langchain-embedding_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +### Standard Usage + +The below example shows how to perform a basic similarity search with Supabase: + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromTexts( + ["Hello world", "Bye bye", "What''s this?"], + [{ id: 2 }, { id: 1 }, { id: 3 }], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + const resultOne = await vectorStore.similaritySearch("Hello world", 1); + + console.log(resultOne); +}; +``` + +### Metadata Filtering + +Given the above `match_documents` Postgres function, you can also pass a filter parameter to only documents with a specific metadata field value. + +```js +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; + +// First, follow set-up instructions at +// https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromTexts( + ["Hello world", "Hello world", "Hello world"], + [{ user_id: 2 }, { user_id: 1 }, { user_id: 3 }], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + const result = await vectorStore.similaritySearch("Hello world", 1, { + user_id: 3, + }); + + console.log(result); +}; +``` + +For more details, checkout the LangChain Supabase integration docs: https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175952_langchain-embedding_search-1_1_0_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175952_langchain-embedding_search-1_1_0_80.snap new file mode 100644 index 000000000..e27d04d07 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175952_langchain-embedding_search-1_1_0_80.snap @@ -0,0 +1,184 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230508175952_langchain-embedding_search-1_1_0.sql +--- +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_name = + 'langchain-embedding_search' + ), + (1, 1, 0), + ' +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''vector'' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception ''"langchain-embedding_search" requires "vector"'' + using hint = ''Run "create extension vector" and try again''; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int, + filter jsonb DEFAULT ''{}'' +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + where metadata @> filter + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +', + ' +# embedding_search + +[LangChain](https://js.langchain.com/docs/) is a framework for developing applications powered by language models with a plugable architecture. + +`langchain-embedding_search` uses a Supabase Postgres database as its vector store. + +## Installation + +```sql +select dbdev.install(''langchain-embedding_search''); +create extension if not exists vector; +create extension "langchain-embedding_search" + schema public + version ''1.1.0''; +``` +Note: + +`vector` is a dependency of `langchain-embedding_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +### Standard Usage + +The below example shows how to perform a basic similarity search with Supabase: + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromTexts( + ["Hello world", "Bye bye", "What''s this?"], + [{ id: 2 }, { id: 1 }, { id: 3 }], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + const resultOne = await vectorStore.similaritySearch("Hello world", 1); + + console.log(resultOne); +}; +``` + +### Metadata Filtering + +Given the above `match_documents` Postgres function, you can also pass a filter parameter to only documents with a specific metadata field value. + +```js +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; + +// First, follow set-up instructions at +// https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromTexts( + ["Hello world", "Hello world", "Hello world"], + [{ user_id: 2 }, { user_id: 1 }, { user_id: 3 }], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + const result = await vectorStore.similaritySearch("Hello world", 1, { + user_id: 3, + }); + + console.log(result); +}; +``` + +For more details, checkout the LangChain Supabase integration docs: https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175953_langchain-hybrid_search-1_1_0_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175953_langchain-hybrid_search-1_1_0_100.snap new file mode 100644 index 000000000..17afaf4e8 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175953_langchain-hybrid_search-1_1_0_100.snap @@ -0,0 +1,155 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230508175953_langchain-hybrid_search-1_1_0.sql +--- +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_name = 'langchain-hybrid_search' + ), + (1, 1, 0), + ' +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''vector'' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception ''"langchain-hybrid_search" requires "vector"'' + using hint = ''Run "create extension vector" and try again''; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to similarity search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int, + filter jsonb DEFAULT ''{}'' +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + where metadata @> filter + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +-- Create a function to keyword search for documents +create function kw_match_documents(query_text text, match_count int) +returns table (id bigint, content text, metadata jsonb, similarity real) +as $$ + +begin +return query execute +format('' + select + id, content, metadata, ts_rank(to_tsvector(content), plainto_tsquery($1)) as similarity + from + documents + where + to_tsvector(content) @@ plainto_tsquery($1) + order by + similarity desc + limit $2 +'') +using query_text, match_count; +end; +$$ language plpgsql; + +', + ' +# hybrid_search + +Langchain supports hybrid search with a Supabase Postgres database. The hybrid search combines the postgres pgvector extension (similarity search) and Full-Text Search (keyword search) to retrieve documents. You can add documents via SupabaseVectorStore addDocuments function. SupabaseHybridKeyWordSearch accepts embedding, supabase client, number of results for similarity search, and number of results for keyword search as parameters. The getRelevantDocuments function produces a list of documents that has duplicates removed and is sorted by relevance score. + +## Installation + +```sql +select dbdev.install(''langchain-hybrid_search''); +create extension if not exists vector; +create extension "langchain-hybrid_search" + schema public + version ''1.1.0''; +``` +Note: + +`vector` is a dependency of `langchain-hybrid_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; +import { SupabaseHybridSearch } from "langchain/retrievers/supabase"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const embeddings = new OpenAIEmbeddings(); + + const retriever = new SupabaseHybridSearch(embeddings, { + client, + // Below are the defaults, expecting that you set up your supabase table and functions according to the guide above. Please change if necessary. + similarityK: 2, + keywordK: 2, + tableName: "documents", + similarityQueryName: "match_documents", + keywordQueryName: "kw_match_documents", + }); + + const results = await retriever.getRelevantDocuments("hello bye"); + + console.log(results); +}; +``` + +For more details, checkout the LangChain Supabase Hybrid Search docs: https://js.langchain.com/docs/modules/indexes/retrievers/supabase-hybrid +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175953_langchain-hybrid_search-1_1_0_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175953_langchain-hybrid_search-1_1_0_80.snap new file mode 100644 index 000000000..c43dc4bf6 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230508175953_langchain-hybrid_search-1_1_0_80.snap @@ -0,0 +1,160 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230508175953_langchain-hybrid_search-1_1_0.sql +--- +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_name = 'langchain-hybrid_search' + ), + (1, 1, 0), + ' +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''vector'' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception ''"langchain-hybrid_search" requires "vector"'' + using hint = ''Run "create extension vector" and try again''; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to similarity search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int, + filter jsonb DEFAULT ''{}'' +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + where metadata @> filter + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +-- Create a function to keyword search for documents +create function kw_match_documents(query_text text, match_count int) +returns table (id bigint, content text, metadata jsonb, similarity real) +as $$ + +begin +return query execute +format('' + select + id, content, metadata, ts_rank(to_tsvector(content), plainto_tsquery($1)) as similarity + from + documents + where + to_tsvector(content) @@ plainto_tsquery($1) + order by + similarity desc + limit $2 +'') +using query_text, match_count; +end; +$$ language plpgsql; + +', + ' +# hybrid_search + +Langchain supports hybrid search with a Supabase Postgres database. The hybrid search combines the postgres pgvector extension (similarity search) and Full-Text Search (keyword search) to retrieve documents. You can add documents via SupabaseVectorStore addDocuments function. SupabaseHybridKeyWordSearch accepts embedding, supabase client, number of results for similarity search, and number of results for keyword search as parameters. The getRelevantDocuments function produces a list of documents that has duplicates removed and is sorted by relevance score. + +## Installation + +```sql +select dbdev.install(''langchain-hybrid_search''); +create extension if not exists vector; +create extension "langchain-hybrid_search" + schema public + version ''1.1.0''; +``` +Note: + +`vector` is a dependency of `langchain-hybrid_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; +import { SupabaseHybridSearch } from "langchain/retrievers/supabase"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const embeddings = new OpenAIEmbeddings(); + + const retriever = new SupabaseHybridSearch(embeddings, { + client, + // Below are the defaults, expecting that you set up your supabase table and functions according to the guide above. Please change if necessary. + similarityK: 2, + keywordK: 2, + tableName: "documents", + similarityQueryName: "match_documents", + keywordQueryName: "kw_match_documents", + }); + + const results = await retriever.getRelevantDocuments("hello bye"); + + console.log(results); +}; +``` + +For more details, checkout the LangChain Supabase Hybrid Search docs: https://js.langchain.com/docs/modules/indexes/retrievers/supabase-hybrid +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230622212339_langchain_headerkit_config_dump_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230622212339_langchain_headerkit_config_dump_100.snap new file mode 100644 index 000000000..fc5a834ba --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230622212339_langchain_headerkit_config_dump_100.snap @@ -0,0 +1,79 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230622212339_langchain_headerkit_config_dump.sql +--- +update app.packages set control_requires = '{vector}' where handle = 'langchain'; + +insert into app.package_upgrades ( + package_id, + from_version_struct, + to_version_struct, + sql +) +values + ( + ( + select + id + from + app.packages + where + handle = 'langchain' and + partial_name = 'embedding_search' + ), + (1, 1, 0), + (1, 1, 1), + ' +SELECT pg_extension_config_dump(''documents'', ''''); +SELECT pg_extension_config_dump(''documents_id_seq'', ''''); +' + ); + +insert into app.package_upgrades ( + package_id, + from_version_struct, + to_version_struct, + sql +) +values + ( + ( + select + id + from + app.packages + where + handle = 'langchain' and + partial_name = 'hybrid_search' + ), + (1, 1, 0), + (1, 1, 1), + ' +SELECT pg_extension_config_dump(''documents'', ''''); +SELECT pg_extension_config_dump(''documents_id_seq'', ''''); +' + ); + +insert into app.package_upgrades ( + package_id, + from_version_struct, + to_version_struct, + sql +) +values + ( + ( + select + id + from + app.packages + where + partial_name = 'pg_headerkit' + ), + (1, 0, 0), + (1, 0, 1), + ' +SELECT pg_extension_config_dump(''hdr.allow_list'', ''''); +SELECT pg_extension_config_dump(''hdr.deny_list'', ''''); +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230622212339_langchain_headerkit_config_dump_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230622212339_langchain_headerkit_config_dump_80.snap new file mode 100644 index 000000000..bad14ac03 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230622212339_langchain_headerkit_config_dump_80.snap @@ -0,0 +1,82 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230622212339_langchain_headerkit_config_dump.sql +--- +update app.packages +set control_requires = '{vector}' +where + handle = 'langchain'; + +insert into app.package_upgrades ( + package_id, + from_version_struct, + to_version_struct, + sql +) +values + ( + ( + select + id + from + app.packages + where + handle = 'langchain' and + partial_name = 'embedding_search' + ), + (1, 1, 0), + (1, 1, 1), + ' +SELECT pg_extension_config_dump(''documents'', ''''); +SELECT pg_extension_config_dump(''documents_id_seq'', ''''); +' + ); + +insert into app.package_upgrades ( + package_id, + from_version_struct, + to_version_struct, + sql +) +values + ( + ( + select + id + from + app.packages + where + handle = 'langchain' and + partial_name = 'hybrid_search' + ), + (1, 1, 0), + (1, 1, 1), + ' +SELECT pg_extension_config_dump(''documents'', ''''); +SELECT pg_extension_config_dump(''documents_id_seq'', ''''); +' + ); + +insert into app.package_upgrades ( + package_id, + from_version_struct, + to_version_struct, + sql +) +values + ( + ( + select + id + from + app.packages + where + partial_name = 'pg_headerkit' + ), + (1, 0, 0), + (1, 0, 1), + ' +SELECT pg_extension_config_dump(''hdr.allow_list'', ''''); +SELECT pg_extension_config_dump(''hdr.deny_list'', ''''); +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230623181432_dbdev_supports_multiple_versions_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230623181432_dbdev_supports_multiple_versions_100.snap new file mode 100644 index 000000000..860571ff1 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230623181432_dbdev_supports_multiple_versions_100.snap @@ -0,0 +1,296 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230623181432_dbdev_supports_multiple_versions.sql +--- +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_name = 'supabase-dbdev' + ), + (0, 0, 3), + ' + +create schema dbdev; + +create or replace function dbdev.install(package_name text) + returns bool + language plpgsql +as $$ +declare + -- Endpoint + base_url text = ''https://api.database.dev/rest/v1/''; + apikey text = ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyYndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzIsImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJrzM0AQKsu_5k134s''; + + http_ext_schema regnamespace = extnamespace::regnamespace from pg_catalog.pg_extension where extname = ''http'' limit 1; + pgtle_is_available bool = true from pg_catalog.pg_extension where extname = ''pg_tle'' limit 1; + -- HTTP respones + rec jsonb; + status int; + contents json; + + -- Install Record + rec_sql text; + rec_ver text; + rec_from_ver text; + rec_to_ver text; + rec_package_name text; + rec_description text; + rec_requires text[]; +begin + + if http_ext_schema is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the http extension and it is not available''); + end if; + + if pgtle_is_available is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the pgtle extension and it is not available''); + end if; + + ------------------- + -- Base Versions -- + ------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_versions?select=package_name,version,sql,control_description,control_requires&limit=50&package_name=eq.%s'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading versions from dbdev''); + end if; + + if contents is null or json_typeof(contents) <> ''array'' or json_array_length(contents) = 0 then + raise exception using errcode=''22000'', message=format(''No versions for package named named %s'', package_name); + end if; + + for rec_package_name, rec_ver, rec_sql, rec_description, rec_requires in select + (r ->> ''package_name''), + (r ->> ''version''), + (r ->> ''sql''), + (r ->> ''control_description''), + array(select json_array_elements_text((r -> ''control_requires''))) + from + json_array_elements(contents) as r + loop + + -- Install the primary version + if not exists ( + select true + from pgtle.available_extensions() + where + name = rec_package_name + ) then + perform pgtle.install_extension(rec_package_name, rec_ver, rec_package_name, rec_sql, rec_requires); + end if; + + -- Install other available versions + if not exists ( + select true + from pgtle.available_extension_versions() + where + name = rec_package_name + and version = rec_ver + ) then + perform pgtle.install_extension_version_sql(rec_package_name, rec_ver, rec_sql); + end if; + + end loop; + + ---------------------- + -- Upgrade Versions -- + ---------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_upgrades?select=package_name,from_version,to_version,sql&limit=50&package_name=eq.%s'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading upgrade pathes from dbdev''); + end if; + + if json_typeof(contents) <> ''array'' then + raise exception using errcode=''22000'', message=format(''Invalid response from dbdev upgrade pathes''); + end if; + + for rec_package_name, rec_from_ver, rec_to_ver, rec_sql in select + (r ->> ''package_name''), + (r ->> ''from_version''), + (r ->> ''to_version''), + (r ->> ''sql'') + from + json_array_elements(contents) as r + loop + + if not exists ( + select true + from pgtle.extension_update_paths(rec_package_name) + where + source = rec_from_ver + and target = rec_to_ver + and path is not null + ) then + perform pgtle.install_update_path(rec_package_name, rec_from_ver, rec_to_ver, rec_sql); + end if; + end loop; + + -------------------------- + -- Send Download Notice -- + -------------------------- + -- Notifies dbdev that a package has been downloaded and records IP + user agent so we can compute unique download counts + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''POST'', + format( + ''%srpc/register_download'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header, + (''x-client-info'', ''dbdev/0.0.2'')::http_header + ], + ''application/json'', + json_build_object(''package_name'', $stmt$ || pg_catalog.quote_literal($1) || $stmt$)::text + ) + ) x + limit 1; $stmt$ + into rec; + + return true; +end; +$$; + +', + ' +# dbdev + +dbdev is the SQL client for database.new and is the primary way end users interact with the package (pglet) registry. + +dbdev can be used to load packages from the registry. For example: + +```sql +-- Load the package from the package index +select dbdev.install(''olirice-index_advisor''); +``` +Where `olirice` is the handle of the author and `index_advisor` is the name of the pglet. + +Once installed, pglets are visible in PostgreSQL as extensions. At that point they can be enabled with standard Postgres commands i.e. the `create extension` + +To improve reproducibility, we recommend __always__ specifying the package version in your `create extension` statements. + +For example: +```sql +-- Enable the extension +create extension "olirice-index_advisor" + schema ''public'' + version ''0.1.0''; +``` + +Which creates all tables/indexes/functions/etc specified by the extension. + +## How to Install + +The in-database SQL client for the package registry is named `dbdev`. You can bootstrap the client with: + +```sql +/*--------------------- +---- install dbdev ---- +---------------------- +Requires: + - pg_tle: https://github.com/aws/pg_tle + - pgsql-http: https://github.com/pramsey/pgsql-http +*/ +create extension if not exists http with schema extensions; +create extension if not exists pg_tle; +select pgtle.uninstall_extension_if_exists(''supabase-dbdev''); +drop extension if exists "supabase-dbdev"; +select + pgtle.install_extension( + ''supabase-dbdev'', + resp.contents ->> ''version'', + ''PostgreSQL package manager'', + resp.contents ->> ''sql'' + ) +from http( + ( + ''GET'', + ''https://api.database.dev/rest/v1/'' + || ''package_versions?select=sql,version'' + || ''&package_name=eq.supabase-dbdev'' + || ''&order=version.desc'' + || ''&limit=1'', + array[ + ( + ''apiKey'', + ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJp'' + || ''c3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyY'' + || ''ndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzI'' + || ''sImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJ'' + || ''rzM0AQKsu_5k134s'' + )::http_header + ], + null, + null + ) +) x, +lateral ( + select + ((row_to_json(x) -> ''content'') #>> ''{}'')::json -> 0 +) resp(contents); +create extension "supabase-dbdev"; +select dbdev.install(''supabase-dbdev''); +drop extension if exists "supabase-dbdev"; +create extension "supabase-dbdev"; +``` + +With the client ready, search for packages on [database.dev](database.dev) and install them with + +```sql +select dbdev.install(''handle-package_name''); +create extension "handle-package_name" + schema ''public'' + version ''1.2.3''; +``` +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230623181432_dbdev_supports_multiple_versions_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230623181432_dbdev_supports_multiple_versions_80.snap new file mode 100644 index 000000000..7cdd0ef66 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230623181432_dbdev_supports_multiple_versions_80.snap @@ -0,0 +1,301 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230623181432_dbdev_supports_multiple_versions.sql +--- +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_name = 'supabase-dbdev' + ), + (0, 0, 3), + ' + +create schema dbdev; + +create or replace function dbdev.install(package_name text) + returns bool + language plpgsql +as $$ +declare + -- Endpoint + base_url text = ''https://api.database.dev/rest/v1/''; + apikey text = ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyYndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzIsImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJrzM0AQKsu_5k134s''; + + http_ext_schema regnamespace = extnamespace::regnamespace from pg_catalog.pg_extension where extname = ''http'' limit 1; + pgtle_is_available bool = true from pg_catalog.pg_extension where extname = ''pg_tle'' limit 1; + -- HTTP respones + rec jsonb; + status int; + contents json; + + -- Install Record + rec_sql text; + rec_ver text; + rec_from_ver text; + rec_to_ver text; + rec_package_name text; + rec_description text; + rec_requires text[]; +begin + + if http_ext_schema is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the http extension and it is not available''); + end if; + + if pgtle_is_available is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the pgtle extension and it is not available''); + end if; + + ------------------- + -- Base Versions -- + ------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_versions?select=package_name,version,sql,control_description,control_requires&limit=50&package_name=eq.%s'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading versions from dbdev''); + end if; + + if contents is null or json_typeof(contents) <> ''array'' or json_array_length(contents) = 0 then + raise exception using errcode=''22000'', message=format(''No versions for package named named %s'', package_name); + end if; + + for rec_package_name, rec_ver, rec_sql, rec_description, rec_requires in select + (r ->> ''package_name''), + (r ->> ''version''), + (r ->> ''sql''), + (r ->> ''control_description''), + array(select json_array_elements_text((r -> ''control_requires''))) + from + json_array_elements(contents) as r + loop + + -- Install the primary version + if not exists ( + select true + from pgtle.available_extensions() + where + name = rec_package_name + ) then + perform pgtle.install_extension(rec_package_name, rec_ver, rec_package_name, rec_sql, rec_requires); + end if; + + -- Install other available versions + if not exists ( + select true + from pgtle.available_extension_versions() + where + name = rec_package_name + and version = rec_ver + ) then + perform pgtle.install_extension_version_sql(rec_package_name, rec_ver, rec_sql); + end if; + + end loop; + + ---------------------- + -- Upgrade Versions -- + ---------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_upgrades?select=package_name,from_version,to_version,sql&limit=50&package_name=eq.%s'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading upgrade pathes from dbdev''); + end if; + + if json_typeof(contents) <> ''array'' then + raise exception using errcode=''22000'', message=format(''Invalid response from dbdev upgrade pathes''); + end if; + + for rec_package_name, rec_from_ver, rec_to_ver, rec_sql in select + (r ->> ''package_name''), + (r ->> ''from_version''), + (r ->> ''to_version''), + (r ->> ''sql'') + from + json_array_elements(contents) as r + loop + + if not exists ( + select true + from pgtle.extension_update_paths(rec_package_name) + where + source = rec_from_ver + and target = rec_to_ver + and path is not null + ) then + perform pgtle.install_update_path(rec_package_name, rec_from_ver, rec_to_ver, rec_sql); + end if; + end loop; + + -------------------------- + -- Send Download Notice -- + -------------------------- + -- Notifies dbdev that a package has been downloaded and records IP + user agent so we can compute unique download counts + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''POST'', + format( + ''%srpc/register_download'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(apikey) || $stmt$)::http_header, + (''x-client-info'', ''dbdev/0.0.2'')::http_header + ], + ''application/json'', + json_build_object(''package_name'', $stmt$ || pg_catalog.quote_literal($1) || $stmt$)::text + ) + ) x + limit 1; $stmt$ + into rec; + + return true; +end; +$$; + +', + ' +# dbdev + +dbdev is the SQL client for database.new and is the primary way end users interact with the package (pglet) registry. + +dbdev can be used to load packages from the registry. For example: + +```sql +-- Load the package from the package index +select dbdev.install(''olirice-index_advisor''); +``` +Where `olirice` is the handle of the author and `index_advisor` is the name of the pglet. + +Once installed, pglets are visible in PostgreSQL as extensions. At that point they can be enabled with standard Postgres commands i.e. the `create extension` + +To improve reproducibility, we recommend __always__ specifying the package version in your `create extension` statements. + +For example: +```sql +-- Enable the extension +create extension "olirice-index_advisor" + schema ''public'' + version ''0.1.0''; +``` + +Which creates all tables/indexes/functions/etc specified by the extension. + +## How to Install + +The in-database SQL client for the package registry is named `dbdev`. You can bootstrap the client with: + +```sql +/*--------------------- +---- install dbdev ---- +---------------------- +Requires: + - pg_tle: https://github.com/aws/pg_tle + - pgsql-http: https://github.com/pramsey/pgsql-http +*/ +create extension if not exists http with schema extensions; +create extension if not exists pg_tle; +select pgtle.uninstall_extension_if_exists(''supabase-dbdev''); +drop extension if exists "supabase-dbdev"; +select + pgtle.install_extension( + ''supabase-dbdev'', + resp.contents ->> ''version'', + ''PostgreSQL package manager'', + resp.contents ->> ''sql'' + ) +from http( + ( + ''GET'', + ''https://api.database.dev/rest/v1/'' + || ''package_versions?select=sql,version'' + || ''&package_name=eq.supabase-dbdev'' + || ''&order=version.desc'' + || ''&limit=1'', + array[ + ( + ''apiKey'', + ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJp'' + || ''c3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyY'' + || ''ndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzI'' + || ''sImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJ'' + || ''rzM0AQKsu_5k134s'' + )::http_header + ], + null, + null + ) +) x, +lateral ( + select + ((row_to_json(x) -> ''content'') #>> ''{}'')::json -> 0 +) resp(contents); +create extension "supabase-dbdev"; +select dbdev.install(''supabase-dbdev''); +drop extension if exists "supabase-dbdev"; +create extension "supabase-dbdev"; +``` + +With the client ready, search for packages on [database.dev](database.dev) and install them with + +```sql +select dbdev.install(''handle-package_name''); +create extension "handle-package_name" + schema ''public'' + version ''1.2.3''; +``` +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230829125510_fix_view_permissions_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230829125510_fix_view_permissions_100.snap new file mode 100644 index 000000000..c9eb1b0ab --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230829125510_fix_view_permissions_100.snap @@ -0,0 +1,21 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230829125510_fix_view_permissions.sql +--- +alter view public.accounts + set (security_invoker = 'true'); + +alter view public.organizations + set (security_invoker = 'true'); + +alter view public.members + set (security_invoker = 'true'); + +alter view public.packages + set (security_invoker = 'true'); + +alter view public.package_versions + set (security_invoker = 'true'); + +alter view public.package_upgrades + set (security_invoker = 'true'); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230829125510_fix_view_permissions_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230829125510_fix_view_permissions_80.snap new file mode 100644 index 000000000..c9eb1b0ab --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230829125510_fix_view_permissions_80.snap @@ -0,0 +1,21 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230829125510_fix_view_permissions.sql +--- +alter view public.accounts + set (security_invoker = 'true'); + +alter view public.organizations + set (security_invoker = 'true'); + +alter view public.members + set (security_invoker = 'true'); + +alter view public.packages + set (security_invoker = 'true'); + +alter view public.package_versions + set (security_invoker = 'true'); + +alter view public.package_upgrades + set (security_invoker = 'true'); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230830083255_olirice-index_advisor-0_2_0_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230830083255_olirice-index_advisor-0_2_0_100.snap new file mode 100644 index 000000000..841d3bfcc --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230830083255_olirice-index_advisor-0_2_0_100.snap @@ -0,0 +1,354 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230830083255_olirice-index_advisor-0_2_0.sql +--- +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_name = 'olirice-index_advisor' + ), + (0, 2, 0), + ' + +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + hypopg_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''hypopg'' + and installed_version is not null + ); + begin + + if not hypopg_exists then + raise + exception ''"olirice-index_advisor" requires "hypopg"'' + using hint = ''Run "create extension hypopg" and try again''; + end if; + end +$$; + +create or replace function index_advisor( + query text +) + returns table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[], + errors text[] + ) + volatile + language plpgsql + as $$ +declare + n_args int; + prepared_statement_name text = ''index_advisor_working_statement''; + hypopg_schema_name text = (select extnamespace::regnamespace::text from pg_extension where extname = ''hypopg''); + explain_plan_statement text; + error_message text; + rec record; + plan_initial jsonb; + plan_final jsonb; + statements text[] = ''{}''; +begin + + -- Remove comment lines (its common that they contain semicolons) + query := trim( + regexp_replace( + regexp_replace( + regexp_replace(query,''\/\*.+\*\/'', '''', ''g''), + ''--[^\r\n]*'', '' '', ''g''), + ''\s+'', '' '', ''g'') + ); + + -- Remove trailing semicolon + query := regexp_replace(query, '';\s*$'', ''''); + + begin + -- Disallow multiple statements + if query ilike ''%;%'' then + raise exception ''Query must not contain a semicolon''; + end if; + + -- Hack to support PostgREST because the prepared statement for args incorrectly defaults to text + query := replace(query, ''WITH pgrst_payload AS (SELECT $1 AS json_data)'', ''WITH pgrst_payload AS (SELECT $1::json AS json_data)''); + + -- Create a prepared statement for the given query + deallocate all; + execute format(''prepare %I as %s'', prepared_statement_name, query); + + -- Detect how many arguments are present in the prepared statement + n_args = ( + select + coalesce(array_length(parameter_types, 1), 0) + from + pg_prepared_statements + where + name = prepared_statement_name + limit + 1 + ); + + -- Create a SQL statement that can be executed to collect the explain plan + explain_plan_statement = format( + ''set local plan_cache_mode = force_generic_plan; explain (format json) execute %I%s'', + --''explain (format json) execute %I%s'', + prepared_statement_name, + case + when n_args = 0 then '''' + else format( + ''(%s)'', array_to_string(array_fill(''null''::text, array[n_args]), '','') + ) + end + ); + + -- Store the query plan before any new indexes + execute explain_plan_statement into plan_initial; + + -- Create possible indexes + for rec in ( + with extension_regclass as ( + select + distinct objid as oid + from + pg_catalog.pg_depend + where + deptype = ''e'' + ) + select + pc.relnamespace::regnamespace::text as schema_name, + pc.relname as table_name, + pa.attname as column_name, + format( + ''select %I.hypopg_create_index($i$create index on %I.%I(%I)$i$)'', + hypopg_schema_name, + pc.relnamespace::regnamespace::text, + pc.relname, + pa.attname + ) hypopg_statement + from + pg_catalog.pg_class pc + join pg_catalog.pg_attribute pa + on pc.oid = pa.attrelid + left join extension_regclass er + on pc.oid = er.oid + left join pg_catalog.pg_index pi + on pc.oid = pi.indrelid + and (select array_agg(x) from unnest(pi.indkey) v(x)) = array[pa.attnum] + and pi.indexprs is null -- ignore expression indexes + and pi.indpred is null -- ignore partial indexes + where + pc.relnamespace::regnamespace::text not in ( -- ignore schema list + ''pg_catalog'', ''pg_toast'', ''information_schema'' + ) + and er.oid is null -- ignore entities owned by extensions + and pc.relkind in (''r'', ''m'') -- regular tables, and materialized views + and pc.relpersistence = ''p'' -- permanent tables (not unlogged or temporary) + and pa.attnum > 0 + and not pa.attisdropped + and pi.indrelid is null + and pa.atttypid in (20,16,1082,1184,1114,701,23,21,700,1083,2950,1700,25,18,1042,1043) + ) + loop + -- Create the hypothetical index + execute rec.hypopg_statement; + end loop; + + -- Create a prepared statement for the given query + -- The original prepared statement MUST be dropped because its plan is cached + execute format(''deallocate %I'', prepared_statement_name); + execute format(''prepare %I as %s'', prepared_statement_name, query); + + -- Store the query plan after new indexes + execute explain_plan_statement into plan_final; + + --raise notice ''%'', plan_final; + + -- Idenfity referenced indexes in new plan + execute format( + ''select + coalesce(array_agg(hypopg_get_indexdef(indexrelid) order by indrelid, indkey::text), $i${}$i$::text[]) + from + %I.hypopg() + where + %s ilike ($i$%%$i$ || indexname || $i$%%$i$) + '', + hypopg_schema_name, + quote_literal(plan_final)::text + ) into statements; + + -- Reset all hypothetical indexes + perform hypopg_reset(); + + -- Reset prepared statements + deallocate all; + + return query values ( + (plan_initial -> 0 -> ''Plan'' -> ''Startup Cost''), + (plan_final -> 0 -> ''Plan'' -> ''Startup Cost''), + (plan_initial -> 0 -> ''Plan'' -> ''Total Cost''), + (plan_final -> 0 -> ''Plan'' -> ''Total Cost''), + statements::text[], + array[]::text[] + ); + return; + + exception when others then + get stacked diagnostics error_message = MESSAGE_TEXT; + + return query values ( + null::jsonb, + null::jsonb, + null::jsonb, + null::jsonb, + array[]::text[], + array[error_message]::text[] + ); + return; + end; + +end; +$$; + +', + ' + +# index_advisor + +`index_advisor` is an extension for recommending indexes to improve query performance. + +## Installation + +Note: + +`hypopg` is a dependency of index_advisor. +Dependency resolution is currently under development. +In the future it will not be necessary to manually create dependencies. + + +```sql +select dbdev.install(''olirice-index_advisor''); +create extension if not exists hypopg; +create extension "olirice-index_advisor" version ''0.2.0''; +``` + +Features: +- Supports generic parameters e.g. `$1`, `$2` +- Supports materialized views +- Identifies tables/columns obfuscaed by views + + +## API + +#### Description +For a given *query*, searches for a set of SQL DDL `create index` statements that improve the query''s execution time; + +#### Signature +```sql +index_advisor(query text) +returns + table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[], + errors text[] + ) +``` + +## Usage + +For a minimal example, the `index_advisor` function can be given a single table query with a filter on an unindexed column. + +```sql +create extension if not exists index_advisor cascade; + +create table book( + id int primary key, + title text not null +); + +select + * +from + index_advisor(''select book.id from book where title = $1''); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements | errors +---------------------+--------------------+-------------------+------------------+-----------------------------------------------------+-------- + 0.00 | 1.17 | 25.88 | 6.40 | {"CREATE INDEX ON public.book USING btree (title)"},| {} +(1 row) +``` + +More complex queries may generate additional suggested indexes + +```sql +create extension if not exists index_advisor cascade; + +create table author( + id serial primary key, + name text not null +); + +create table publisher( + id serial primary key, + name text not null, + corporate_address text +); + +create table book( + id serial primary key, + author_id int not null references author(id), + publisher_id int not null references publisher(id), + title text +); + +create table review( + id serial primary key, + book_id int references book(id), + body text not null +); + +select + * +from + index_advisor('' + select + book.id, + book.title, + publisher.name as publisher_name, + author.name as author_name, + review.body review_body + from + book + join publisher + on book.publisher_id = publisher.id + join author + on book.author_id = author.id + join review + on book.id = review.book_id + where + author.id = $1 + and publisher.id = $2 + ''); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements | errors +---------------------+--------------------+-------------------+------------------+-----------------------------------------------------------+-------- + 27.26 | 12.77 | 68.48 | 42.37 | {"CREATE INDEX ON public.book USING btree (author_id)", | {} + "CREATE INDEX ON public.book USING btree (publisher_id)", + "CREATE INDEX ON public.review USING btree (book_id)"} +(3 rows) +``` +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230830083255_olirice-index_advisor-0_2_0_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230830083255_olirice-index_advisor-0_2_0_80.snap new file mode 100644 index 000000000..2c5aafa32 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230830083255_olirice-index_advisor-0_2_0_80.snap @@ -0,0 +1,359 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230830083255_olirice-index_advisor-0_2_0.sql +--- +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_name = 'olirice-index_advisor' + ), + (0, 2, 0), + ' + +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + hypopg_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''hypopg'' + and installed_version is not null + ); + begin + + if not hypopg_exists then + raise + exception ''"olirice-index_advisor" requires "hypopg"'' + using hint = ''Run "create extension hypopg" and try again''; + end if; + end +$$; + +create or replace function index_advisor( + query text +) + returns table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[], + errors text[] + ) + volatile + language plpgsql + as $$ +declare + n_args int; + prepared_statement_name text = ''index_advisor_working_statement''; + hypopg_schema_name text = (select extnamespace::regnamespace::text from pg_extension where extname = ''hypopg''); + explain_plan_statement text; + error_message text; + rec record; + plan_initial jsonb; + plan_final jsonb; + statements text[] = ''{}''; +begin + + -- Remove comment lines (its common that they contain semicolons) + query := trim( + regexp_replace( + regexp_replace( + regexp_replace(query,''\/\*.+\*\/'', '''', ''g''), + ''--[^\r\n]*'', '' '', ''g''), + ''\s+'', '' '', ''g'') + ); + + -- Remove trailing semicolon + query := regexp_replace(query, '';\s*$'', ''''); + + begin + -- Disallow multiple statements + if query ilike ''%;%'' then + raise exception ''Query must not contain a semicolon''; + end if; + + -- Hack to support PostgREST because the prepared statement for args incorrectly defaults to text + query := replace(query, ''WITH pgrst_payload AS (SELECT $1 AS json_data)'', ''WITH pgrst_payload AS (SELECT $1::json AS json_data)''); + + -- Create a prepared statement for the given query + deallocate all; + execute format(''prepare %I as %s'', prepared_statement_name, query); + + -- Detect how many arguments are present in the prepared statement + n_args = ( + select + coalesce(array_length(parameter_types, 1), 0) + from + pg_prepared_statements + where + name = prepared_statement_name + limit + 1 + ); + + -- Create a SQL statement that can be executed to collect the explain plan + explain_plan_statement = format( + ''set local plan_cache_mode = force_generic_plan; explain (format json) execute %I%s'', + --''explain (format json) execute %I%s'', + prepared_statement_name, + case + when n_args = 0 then '''' + else format( + ''(%s)'', array_to_string(array_fill(''null''::text, array[n_args]), '','') + ) + end + ); + + -- Store the query plan before any new indexes + execute explain_plan_statement into plan_initial; + + -- Create possible indexes + for rec in ( + with extension_regclass as ( + select + distinct objid as oid + from + pg_catalog.pg_depend + where + deptype = ''e'' + ) + select + pc.relnamespace::regnamespace::text as schema_name, + pc.relname as table_name, + pa.attname as column_name, + format( + ''select %I.hypopg_create_index($i$create index on %I.%I(%I)$i$)'', + hypopg_schema_name, + pc.relnamespace::regnamespace::text, + pc.relname, + pa.attname + ) hypopg_statement + from + pg_catalog.pg_class pc + join pg_catalog.pg_attribute pa + on pc.oid = pa.attrelid + left join extension_regclass er + on pc.oid = er.oid + left join pg_catalog.pg_index pi + on pc.oid = pi.indrelid + and (select array_agg(x) from unnest(pi.indkey) v(x)) = array[pa.attnum] + and pi.indexprs is null -- ignore expression indexes + and pi.indpred is null -- ignore partial indexes + where + pc.relnamespace::regnamespace::text not in ( -- ignore schema list + ''pg_catalog'', ''pg_toast'', ''information_schema'' + ) + and er.oid is null -- ignore entities owned by extensions + and pc.relkind in (''r'', ''m'') -- regular tables, and materialized views + and pc.relpersistence = ''p'' -- permanent tables (not unlogged or temporary) + and pa.attnum > 0 + and not pa.attisdropped + and pi.indrelid is null + and pa.atttypid in (20,16,1082,1184,1114,701,23,21,700,1083,2950,1700,25,18,1042,1043) + ) + loop + -- Create the hypothetical index + execute rec.hypopg_statement; + end loop; + + -- Create a prepared statement for the given query + -- The original prepared statement MUST be dropped because its plan is cached + execute format(''deallocate %I'', prepared_statement_name); + execute format(''prepare %I as %s'', prepared_statement_name, query); + + -- Store the query plan after new indexes + execute explain_plan_statement into plan_final; + + --raise notice ''%'', plan_final; + + -- Idenfity referenced indexes in new plan + execute format( + ''select + coalesce(array_agg(hypopg_get_indexdef(indexrelid) order by indrelid, indkey::text), $i${}$i$::text[]) + from + %I.hypopg() + where + %s ilike ($i$%%$i$ || indexname || $i$%%$i$) + '', + hypopg_schema_name, + quote_literal(plan_final)::text + ) into statements; + + -- Reset all hypothetical indexes + perform hypopg_reset(); + + -- Reset prepared statements + deallocate all; + + return query values ( + (plan_initial -> 0 -> ''Plan'' -> ''Startup Cost''), + (plan_final -> 0 -> ''Plan'' -> ''Startup Cost''), + (plan_initial -> 0 -> ''Plan'' -> ''Total Cost''), + (plan_final -> 0 -> ''Plan'' -> ''Total Cost''), + statements::text[], + array[]::text[] + ); + return; + + exception when others then + get stacked diagnostics error_message = MESSAGE_TEXT; + + return query values ( + null::jsonb, + null::jsonb, + null::jsonb, + null::jsonb, + array[]::text[], + array[error_message]::text[] + ); + return; + end; + +end; +$$; + +', + ' + +# index_advisor + +`index_advisor` is an extension for recommending indexes to improve query performance. + +## Installation + +Note: + +`hypopg` is a dependency of index_advisor. +Dependency resolution is currently under development. +In the future it will not be necessary to manually create dependencies. + + +```sql +select dbdev.install(''olirice-index_advisor''); +create extension if not exists hypopg; +create extension "olirice-index_advisor" version ''0.2.0''; +``` + +Features: +- Supports generic parameters e.g. `$1`, `$2` +- Supports materialized views +- Identifies tables/columns obfuscaed by views + + +## API + +#### Description +For a given *query*, searches for a set of SQL DDL `create index` statements that improve the query''s execution time; + +#### Signature +```sql +index_advisor(query text) +returns + table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[], + errors text[] + ) +``` + +## Usage + +For a minimal example, the `index_advisor` function can be given a single table query with a filter on an unindexed column. + +```sql +create extension if not exists index_advisor cascade; + +create table book( + id int primary key, + title text not null +); + +select + * +from + index_advisor(''select book.id from book where title = $1''); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements | errors +---------------------+--------------------+-------------------+------------------+-----------------------------------------------------+-------- + 0.00 | 1.17 | 25.88 | 6.40 | {"CREATE INDEX ON public.book USING btree (title)"},| {} +(1 row) +``` + +More complex queries may generate additional suggested indexes + +```sql +create extension if not exists index_advisor cascade; + +create table author( + id serial primary key, + name text not null +); + +create table publisher( + id serial primary key, + name text not null, + corporate_address text +); + +create table book( + id serial primary key, + author_id int not null references author(id), + publisher_id int not null references publisher(id), + title text +); + +create table review( + id serial primary key, + book_id int references book(id), + body text not null +); + +select + * +from + index_advisor('' + select + book.id, + book.title, + publisher.name as publisher_name, + author.name as author_name, + review.body review_body + from + book + join publisher + on book.publisher_id = publisher.id + join author + on book.author_id = author.id + join review + on book.id = review.book_id + where + author.id = $1 + and publisher.id = $2 + ''); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements | errors +---------------------+--------------------+-------------------+------------------+-----------------------------------------------------------+-------- + 27.26 | 12.77 | 68.48 | 42.37 | {"CREATE INDEX ON public.book USING btree (author_id)", | {} + "CREATE INDEX ON public.book USING btree (publisher_id)", + "CREATE INDEX ON public.review USING btree (book_id)"} +(3 rows) +``` +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230831172915_allow_anon_access_to_package_views_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230831172915_allow_anon_access_to_package_views_100.snap new file mode 100644 index 000000000..d42c618bf --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230831172915_allow_anon_access_to_package_views_100.snap @@ -0,0 +1,12 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230831172915_allow_anon_access_to_package_views.sql +--- +alter view public.packages + set (security_invoker = 'false'); + +alter view public.package_versions + set (security_invoker = 'false'); + +alter view public.package_upgrades + set (security_invoker = 'false'); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230831172915_allow_anon_access_to_package_views_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230831172915_allow_anon_access_to_package_views_80.snap new file mode 100644 index 000000000..d42c618bf --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230831172915_allow_anon_access_to_package_views_80.snap @@ -0,0 +1,12 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230831172915_allow_anon_access_to_package_views.sql +--- +alter view public.packages + set (security_invoker = 'false'); + +alter view public.package_versions + set (security_invoker = 'false'); + +alter view public.package_upgrades + set (security_invoker = 'false'); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906110845_access_token_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906110845_access_token_100.snap new file mode 100644 index 000000000..f42c32f93 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906110845_access_token_100.snap @@ -0,0 +1,201 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230906110845_access_token.sql +--- +create table app.access_tokens ( + id uuid primary key default gen_random_uuid(), + user_id uuid + not null + references auth.users (id) on DELETE cascade, + token_hash bytea not null, + token_name text + not null + check (length(token_name) <= 64), + plaintext_suffix text + not null + check (length(plaintext_suffix) = 4), + created_at timestamp with time zone + not null + default NOW() +); + +grant INSERT (id, +user_id, +token_hash, +token_name, +plaintext_suffix) +on table app.access_tokens +to authenticated; + +grant DELETE on table app.access_tokens to authenticated; + +alter table app.access_tokens + enable row level security; + +create policy access_tokens_select_policy +on app.access_tokens +as permissive +for select +to public +using (auth.uid() = user_id); + +create policy access_tokens_insert_policy +on app.access_tokens +as permissive +for insert +to authenticated +with check (auth.uid() = user_id); + +create policy access_tokens_delete_policy +on app.access_tokens +as permissive +for delete +to authenticated +using (auth.uid() = user_id); + +create or replace function app.base64url_encode(input bytea) +returns text +language plpgsql +strict +as $function$ +begin + return replace(replace(encode(input, 'base64'), '/', '_'), '+', '-'); +end; +$function$; + +create or replace function app.base64url_decode(input text) +returns text +language plpgsql +strict +as $function$ +begin + return decode(replace(replace(input, '-', '+'), '_', '/'), 'base64'); +end; +$function$; + +create or replace function public.new_access_token(token_name text) +returns text +language plpgsql +strict +as $function$ +<> +declare + account app.accounts = account from app.accounts account where id = auth.uid(); + -- Why 21 random bytes? We are shooting for 128 bit (16 bytes) entropy. But we + -- also show three bytes as plaintext. That takes us to a total 19 bytes. + -- We add two bytes to make sure that the base64 encoded bytes don't have any + -- padding, which makes it a little bit nicer to look at. That makes 21. + token bytea = gen_random_bytes(21); + token_hash bytea = sha256(token); + -- Total length of the base64 encoded token is 21 * 8 / 6 = 28 + token_text text = app.base64url_encode(token); + token_id uuid; + -- Last 4 base64 encoded character are shown in the suffix + plaintext_suffix text = substring(token_text from 25); +begin + insert into app.access_tokens(user_id, token_hash, token_name, plaintext_suffix) + values (account.id, token_hash, token_name, fn.plaintext_suffix) returning id into token_id; + + -- String returned has a length 64 + return 'dbd_' || replace(token_id::text, '-', '') || token_text; +end; +$function$; + +create type app.access_token_struct as ( + id uuid, + token_name text, + masked_token text, + created_at timestamp with time zone +); + +create or replace function public.get_access_tokens() +returns setof app.access_token_struct +language plpgsql +strict +as $function$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); +begin + return query + select id, token_name, + 'dbd_' || + substring(at.id::text from 1 for 4) || + repeat('•', 52) || + at.plaintext_suffix as masked_token, + created_at + from app.access_tokens at + where at.user_id = account.id; +end; +$function$; + +create or replace function public.delete_access_token(token_id uuid) +returns void +language plpgsql +strict +as $function$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); +begin + delete from app.access_tokens at + where at.user_id = account.id and at.id = token_id; +end; +$function$; + +create type app.user_id_and_token_hash as (user_id uuid, token_hash bytea); + +create or replace function public.redeem_access_token(access_token text) +returns text +language plpgsql +strict +security definer +as $function$ +declare + token_id uuid; + token bytea; + tokens_row app.user_id_and_token_hash; + token_valid boolean; + now timestamp; + one_hour_from_now timestamp; + issued_at int; + expiry_at int; + jwt_secret text; +begin + -- validate access token + if length(access_token) != 64 then + raise exception 'Invalid token'; + end if; + + if substring(access_token from 1 for 4) != 'dbd_' then + raise exception 'Invalid token'; + end if; + + token_id := substring(access_token from 5 for 32)::uuid; + token := app.base64url_decode(substring(access_token from 37)); + + select t.user_id, t.token_hash + into tokens_row + from app.access_tokens t + where t.id = token_id; + + -- TODO: do a constant time comparison + if tokens_row.token_hash != sha256(token) then + raise exception 'Invalid token'; + end if; + + -- Generate JWT token + now := current_timestamp; + one_hour_from_now := now + interval '1 hour'; + issued_at := date_part('epoch', now); + expiry_at := date_part('epoch', one_hour_from_now); + jwt_secret := current_setting('app.settings.jwt_secret', true); + + return sign(json_build_object( + 'aud', 'authenticated', + 'role', 'authenticated', + 'iss', 'database.dev', + 'sub', tokens_row.user_id, + 'iat', issued_at, + 'exp', expiry_at + ), jwt_secret); +end; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906110845_access_token_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906110845_access_token_80.snap new file mode 100644 index 000000000..146b25ceb --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906110845_access_token_80.snap @@ -0,0 +1,204 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230906110845_access_token.sql +--- +create table app.access_tokens ( + id uuid + primary key + default gen_random_uuid(), + user_id uuid + not null + references auth.users (id) + on DELETE cascade, + token_hash bytea not null, + token_name text + not null + check (length(token_name) <= 64), + plaintext_suffix text + not null + check (length(plaintext_suffix) = 4), + created_at timestamp with time zone + not null + default NOW() +); + +grant INSERT (id, +user_id, +token_hash, +token_name, +plaintext_suffix) +on table app.access_tokens +to authenticated; + +grant DELETE on table app.access_tokens to authenticated; + +alter table app.access_tokens + enable row level security; + +create policy access_tokens_select_policy +on app.access_tokens +as permissive +for select +to public +using (auth.uid() = user_id); + +create policy access_tokens_insert_policy +on app.access_tokens +as permissive +for insert +to authenticated +with check (auth.uid() = user_id); + +create policy access_tokens_delete_policy +on app.access_tokens +as permissive +for delete +to authenticated +using (auth.uid() = user_id); + +create or replace function app.base64url_encode(input bytea) +returns text +language plpgsql +strict +as $function$ +begin + return replace(replace(encode(input, 'base64'), '/', '_'), '+', '-'); +end; +$function$; + +create or replace function app.base64url_decode(input text) +returns text +language plpgsql +strict +as $function$ +begin + return decode(replace(replace(input, '-', '+'), '_', '/'), 'base64'); +end; +$function$; + +create or replace function public.new_access_token(token_name text) +returns text +language plpgsql +strict +as $function$ +<> +declare + account app.accounts = account from app.accounts account where id = auth.uid(); + -- Why 21 random bytes? We are shooting for 128 bit (16 bytes) entropy. But we + -- also show three bytes as plaintext. That takes us to a total 19 bytes. + -- We add two bytes to make sure that the base64 encoded bytes don't have any + -- padding, which makes it a little bit nicer to look at. That makes 21. + token bytea = gen_random_bytes(21); + token_hash bytea = sha256(token); + -- Total length of the base64 encoded token is 21 * 8 / 6 = 28 + token_text text = app.base64url_encode(token); + token_id uuid; + -- Last 4 base64 encoded character are shown in the suffix + plaintext_suffix text = substring(token_text from 25); +begin + insert into app.access_tokens(user_id, token_hash, token_name, plaintext_suffix) + values (account.id, token_hash, token_name, fn.plaintext_suffix) returning id into token_id; + + -- String returned has a length 64 + return 'dbd_' || replace(token_id::text, '-', '') || token_text; +end; +$function$; + +create type app.access_token_struct as ( + id uuid, + token_name text, + masked_token text, + created_at timestamp with time zone +); + +create or replace function public.get_access_tokens() +returns setof app.access_token_struct +language plpgsql +strict +as $function$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); +begin + return query + select id, token_name, + 'dbd_' || + substring(at.id::text from 1 for 4) || + repeat('•', 52) || + at.plaintext_suffix as masked_token, + created_at + from app.access_tokens at + where at.user_id = account.id; +end; +$function$; + +create or replace function public.delete_access_token(token_id uuid) +returns void +language plpgsql +strict +as $function$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); +begin + delete from app.access_tokens at + where at.user_id = account.id and at.id = token_id; +end; +$function$; + +create type app.user_id_and_token_hash as (user_id uuid, token_hash bytea); + +create or replace function public.redeem_access_token(access_token text) +returns text +language plpgsql +strict +security definer +as $function$ +declare + token_id uuid; + token bytea; + tokens_row app.user_id_and_token_hash; + token_valid boolean; + now timestamp; + one_hour_from_now timestamp; + issued_at int; + expiry_at int; + jwt_secret text; +begin + -- validate access token + if length(access_token) != 64 then + raise exception 'Invalid token'; + end if; + + if substring(access_token from 1 for 4) != 'dbd_' then + raise exception 'Invalid token'; + end if; + + token_id := substring(access_token from 5 for 32)::uuid; + token := app.base64url_decode(substring(access_token from 37)); + + select t.user_id, t.token_hash + into tokens_row + from app.access_tokens t + where t.id = token_id; + + -- TODO: do a constant time comparison + if tokens_row.token_hash != sha256(token) then + raise exception 'Invalid token'; + end if; + + -- Generate JWT token + now := current_timestamp; + one_hour_from_now := now + interval '1 hour'; + issued_at := date_part('epoch', now); + expiry_at := date_part('epoch', one_hour_from_now); + jwt_secret := current_setting('app.settings.jwt_secret', true); + + return sign(json_build_object( + 'aud', 'authenticated', + 'role', 'authenticated', + 'iss', 'database.dev', + 'sub', tokens_row.user_id, + 'iat', issued_at, + 'exp', expiry_at + ), jwt_secret); +end; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906111353_publish_package_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906111353_publish_package_100.snap new file mode 100644 index 000000000..67908582a --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906111353_publish_package_100.snap @@ -0,0 +1,109 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230906111353_publish_package.sql +--- +grant INSERT (partial_name, handle, control_description) on table app.packages to authenticated; + +grant UPDATE (control_description) on table app.packages to authenticated; + +create policy packages_update_policy +on app.packages +as permissive +for update +to authenticated +using (app.is_package_maintainer(auth.uid(), id)); + +create or replace function +public.publish_package( + package_name app.valid_name, + package_description varchar(1000) +) +returns void +language plpgsql +as $function$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); +begin + if account.handle is null then + raise exception 'user not logged in'; + end if; + + insert into app.packages(handle, partial_name, control_description) + values (account.handle, package_name, package_description) + on conflict on constraint packages_handle_partial_name_key + do update + set control_description = excluded.control_description; +end; +$function$; + +create or replace function +public.publish_package_version( + package_name app.valid_name, + version_source text, + version_description text, + version text +) +returns uuid +language plpgsql +as $function$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); + package_id uuid; + version_id uuid; +begin + if account.handle is null then + raise exception 'user not logged in'; + end if; + + select ap.id + from app.packages ap + where ap.handle = account.handle and ap.partial_name = publish_package_version.package_name + into package_id; + + begin + insert into app.package_versions(package_id, version_struct, sql, description_md) + values (package_id, app.text_to_semver(version), version_source, version_description) + returning id into version_id; + + return version_id; + exception when unique_violation then + return null; + end; +end; +$function$; + +create or replace function +public.publish_package_upgrade( + package_name app.valid_name, + upgrade_source text, + from_version text, + to_version text +) +returns uuid +language plpgsql +as $function$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); + package_id uuid; + upgrade_id uuid; +begin + if account.handle is null then + raise exception 'user not logged in'; + end if; + + select ap.id + from app.packages ap + where ap.handle = account.handle and ap.partial_name = publish_package_upgrade.package_name + into package_id; + + begin + insert into app.package_upgrades(package_id, from_version_struct, to_version_struct, sql) + values (package_id, app.text_to_semver(from_version), app.text_to_semver(to_version), upgrade_source) + returning id into upgrade_id; + + return upgrade_id; + exception when unique_violation then + return null; + end; +end; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906111353_publish_package_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906111353_publish_package_80.snap new file mode 100644 index 000000000..955df4be5 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20230906111353_publish_package_80.snap @@ -0,0 +1,116 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20230906111353_publish_package.sql +--- +grant INSERT (partial_name, +handle, +control_description) +on table app.packages +to authenticated; + +grant UPDATE (control_description) on table app.packages to authenticated; + +create policy packages_update_policy +on app.packages +as permissive +for update +to authenticated +using (app.is_package_maintainer( + auth.uid(), + id +)); + +create or replace function +public.publish_package( + package_name app.valid_name, + package_description varchar(1000) +) +returns void +language plpgsql +as $function$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); +begin + if account.handle is null then + raise exception 'user not logged in'; + end if; + + insert into app.packages(handle, partial_name, control_description) + values (account.handle, package_name, package_description) + on conflict on constraint packages_handle_partial_name_key + do update + set control_description = excluded.control_description; +end; +$function$; + +create or replace function +public.publish_package_version( + package_name app.valid_name, + version_source text, + version_description text, + version text +) +returns uuid +language plpgsql +as $function$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); + package_id uuid; + version_id uuid; +begin + if account.handle is null then + raise exception 'user not logged in'; + end if; + + select ap.id + from app.packages ap + where ap.handle = account.handle and ap.partial_name = publish_package_version.package_name + into package_id; + + begin + insert into app.package_versions(package_id, version_struct, sql, description_md) + values (package_id, app.text_to_semver(version), version_source, version_description) + returning id into version_id; + + return version_id; + exception when unique_violation then + return null; + end; +end; +$function$; + +create or replace function +public.publish_package_upgrade( + package_name app.valid_name, + upgrade_source text, + from_version text, + to_version text +) +returns uuid +language plpgsql +as $function$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); + package_id uuid; + upgrade_id uuid; +begin + if account.handle is null then + raise exception 'user not logged in'; + end if; + + select ap.id + from app.packages ap + where ap.handle = account.handle and ap.partial_name = publish_package_upgrade.package_name + into package_id; + + begin + insert into app.package_upgrades(package_id, from_version_struct, to_version_struct, sql) + values (package_id, app.text_to_semver(from_version), app.text_to_semver(to_version), upgrade_source) + returning id into upgrade_id; + + return upgrade_id; + exception when unique_violation then + return null; + end; +end; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231110061036_allow_publishing_relocatable_and_requires_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231110061036_allow_publishing_relocatable_and_requires_100.snap new file mode 100644 index 000000000..ee5c3e8b2 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231110061036_allow_publishing_relocatable_and_requires_100.snap @@ -0,0 +1,171 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231110061036_allow_publishing_relocatable_and_requires.sql +--- +create table app.allowed_extensions (name text primary key); + +insert into app.allowed_extensions (name) +values + ('citext'), + ('pg_cron'), + ('pg_graphql'), + ('pg_stat_statements'), + ('pg_trgm'), + ('pg_crypto'), + ('pg_jwt'), + ('pg_sodium'), + ('plpgsql'), + ('uuid-ossp'), + ('address_standardizer'), + ('address_standardizer_data_us'), + ('autoinc'), + ('bloom'), + ('btree_gin'), + ('btree_gist'), + ('cube'), + ('dblink'), + ('dict_int'), + ('dict_xsyn'), + ('earthdistance'), + ('fuzzystrmatch'), + ('hstore'), + ('http'), + ('hypopg'), + ('insert_username'), + ('intarray'), + ('isn'), + ('ltree'), + ('moddatetime'), + ('pg_hashids'), + ('pg_jsonschema'), + ('pg_net'), + ('pg_repack'), + ('pg_stat_monitor'), + ('pg_walinspect'), + ('pgaudit'), + ('pgroonga'), + ('pgroonga_database'), + ('pgrouting'), + ('pgrowlocks'), + ('pgtap'), + ('plcoffee'), + ('pljava'), + ('plls'), + ('plpgsql_check'), + ('plv8'), + ('postgis'), + ('postgis_raster'), + ('postgis_sfcgal'), + ('postgis_tiger_geocoder'), + ('postgis_topology'), + ('postgres_fdw'), + ('refint'), + ('rum'), + ('seg'), + ('sslinfo'), + ('supautils'), + ('tablefunc'), + ('tcn'), + ('timescaledb'), + ('tsm_system_rows'), + ('tsm_system_time'), + ('unaccent'), + ('vector'), + ('wrappers'), + ('amcheck'), + ('aws_commons'), + ('aws_lambda'), + ('aws_s3'), + ('bool_plperl'), + ('decoder_raw'), + ('h3-pg'), + ('hll'), + ('hstore_plperl'), + ('intagg'), + ('ip4r'), + ('jsonb_plperl'), + ('lo'), + ('log_fdw'), + ('mysql_fdw'), + ('old_snapshot'), + ('oracle_fdw'), + ('orafce'), + ('pageinspect'), + ('pg_bigm'), + ('pg_buffercache'), + ('pg_freespacemap'), + ('pg_hint_plan'), + ('pg_partman'), + ('pg_prewarm'), + ('pg_proctab'), + ('pg_similarity'), + ('pg_tle'), + ('pg_transport'), + ('pg_visibility'), + ('pgcrypto'), + ('pgstattuple'), + ('pgvector'), + ('plperl'), + ('plprofiler'), + ('plrust'), + ('pltcl'), + ('prefix'), + ('rdkit'), + ('rds_tools'), + ('tds_fdw'), + ('test_parser'), + ('wal2json'); + +grant INSERT (partial_name, +handle, +control_description, +control_relocatable, +control_requires) +on table app.packages +to authenticated; + +grant UPDATE (control_description, +control_relocatable, +control_requires) +on table app.packages +to authenticated; + +create or replace function +public.publish_package( + package_name app.valid_name, + package_description varchar(1000), + relocatable boolean default false, + requires text[] default '{}' +) +returns void +language plpgsql +as $function$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); + require text; +begin + if account.handle is null then + raise exception 'user not logged in'; + end if; + + foreach require in array requires + loop + if not exists ( + select true + from app.allowed_extensions + where + name = require + ) then + raise exception '`requires` in the control file can''t have `%` in it', require; + end if; + end loop; + + insert into app.packages(handle, partial_name, control_description, control_relocatable, control_requires) + values (account.handle, package_name, package_description, relocatable, requires) + on conflict on constraint packages_handle_partial_name_key + do update + set control_description = excluded.control_description, + control_relocatable = excluded.control_relocatable, + control_requires = excluded.control_requires; +end; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231110061036_allow_publishing_relocatable_and_requires_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231110061036_allow_publishing_relocatable_and_requires_80.snap new file mode 100644 index 000000000..ee5c3e8b2 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231110061036_allow_publishing_relocatable_and_requires_80.snap @@ -0,0 +1,171 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231110061036_allow_publishing_relocatable_and_requires.sql +--- +create table app.allowed_extensions (name text primary key); + +insert into app.allowed_extensions (name) +values + ('citext'), + ('pg_cron'), + ('pg_graphql'), + ('pg_stat_statements'), + ('pg_trgm'), + ('pg_crypto'), + ('pg_jwt'), + ('pg_sodium'), + ('plpgsql'), + ('uuid-ossp'), + ('address_standardizer'), + ('address_standardizer_data_us'), + ('autoinc'), + ('bloom'), + ('btree_gin'), + ('btree_gist'), + ('cube'), + ('dblink'), + ('dict_int'), + ('dict_xsyn'), + ('earthdistance'), + ('fuzzystrmatch'), + ('hstore'), + ('http'), + ('hypopg'), + ('insert_username'), + ('intarray'), + ('isn'), + ('ltree'), + ('moddatetime'), + ('pg_hashids'), + ('pg_jsonschema'), + ('pg_net'), + ('pg_repack'), + ('pg_stat_monitor'), + ('pg_walinspect'), + ('pgaudit'), + ('pgroonga'), + ('pgroonga_database'), + ('pgrouting'), + ('pgrowlocks'), + ('pgtap'), + ('plcoffee'), + ('pljava'), + ('plls'), + ('plpgsql_check'), + ('plv8'), + ('postgis'), + ('postgis_raster'), + ('postgis_sfcgal'), + ('postgis_tiger_geocoder'), + ('postgis_topology'), + ('postgres_fdw'), + ('refint'), + ('rum'), + ('seg'), + ('sslinfo'), + ('supautils'), + ('tablefunc'), + ('tcn'), + ('timescaledb'), + ('tsm_system_rows'), + ('tsm_system_time'), + ('unaccent'), + ('vector'), + ('wrappers'), + ('amcheck'), + ('aws_commons'), + ('aws_lambda'), + ('aws_s3'), + ('bool_plperl'), + ('decoder_raw'), + ('h3-pg'), + ('hll'), + ('hstore_plperl'), + ('intagg'), + ('ip4r'), + ('jsonb_plperl'), + ('lo'), + ('log_fdw'), + ('mysql_fdw'), + ('old_snapshot'), + ('oracle_fdw'), + ('orafce'), + ('pageinspect'), + ('pg_bigm'), + ('pg_buffercache'), + ('pg_freespacemap'), + ('pg_hint_plan'), + ('pg_partman'), + ('pg_prewarm'), + ('pg_proctab'), + ('pg_similarity'), + ('pg_tle'), + ('pg_transport'), + ('pg_visibility'), + ('pgcrypto'), + ('pgstattuple'), + ('pgvector'), + ('plperl'), + ('plprofiler'), + ('plrust'), + ('pltcl'), + ('prefix'), + ('rdkit'), + ('rds_tools'), + ('tds_fdw'), + ('test_parser'), + ('wal2json'); + +grant INSERT (partial_name, +handle, +control_description, +control_relocatable, +control_requires) +on table app.packages +to authenticated; + +grant UPDATE (control_description, +control_relocatable, +control_requires) +on table app.packages +to authenticated; + +create or replace function +public.publish_package( + package_name app.valid_name, + package_description varchar(1000), + relocatable boolean default false, + requires text[] default '{}' +) +returns void +language plpgsql +as $function$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); + require text; +begin + if account.handle is null then + raise exception 'user not logged in'; + end if; + + foreach require in array requires + loop + if not exists ( + select true + from app.allowed_extensions + where + name = require + ) then + raise exception '`requires` in the control file can''t have `%` in it', require; + end if; + end loop; + + insert into app.packages(handle, partial_name, control_description, control_relocatable, control_requires) + values (account.handle, package_name, package_description, relocatable, requires) + on conflict on constraint packages_handle_partial_name_key + do update + set control_description = excluded.control_description, + control_relocatable = excluded.control_relocatable, + control_requires = excluded.control_requires; +end; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205051816_add_default_version_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205051816_add_default_version_100.snap new file mode 100644 index 000000000..8701dff1e --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205051816_add_default_version_100.snap @@ -0,0 +1,109 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231205051816_add_default_version.sql +--- +alter table app.packages + add column default_version_struct app.semver + not null + default app.text_to_semver('0.0.0'), + add column default_version text + generated always as (app.semver_to_text(default_version_struct)) stored; + +update app.packages +set default_version_struct = app.text_to_semver(pp.latest_version) +from public.packages as pp +where + packages.id = pp.id; + +alter table app.packages + alter column default_version_struct drop default; + +create or replace view public.packages +as select + pa.id, + pa.package_name, + pa.handle, + pa.partial_name, + newest_ver.version as latest_version, + newest_ver.description_md, + pa.control_description, + pa.control_requires, + pa.created_at, + pa.default_version +from + app.packages as pa, + lateral ( + select + * + from + app.package_versions as pv + where + pv.package_id = pa.id + order by pv.version_struct desc + limit 1 + ) + as newest_ver; + +grant INSERT (partial_name, +handle, +control_description, +control_relocatable, +control_requires, +default_version_struct) +on table app.packages +to authenticated; + +grant UPDATE (control_description, +control_relocatable, +control_requires, +default_version_struct) +on table app.packages +to authenticated; + +drop function public.publish_package(app.valid_name, varchar, boolean, text[]); + +create or replace function +public.publish_package( + package_name app.valid_name, + package_description varchar(1000), + relocatable boolean default false, + requires text[] default '{}', + default_version text default null +) +returns void +language plpgsql +as $function$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); + require text; +begin + if account.handle is null then + raise exception 'user not logged in'; + end if; + + if default_version is null then + raise exception 'default_version is required. If you are on `dbdev` CLI version 0.1.5 or older upgrade to the latest version.'; + end if; + + foreach require in array requires + loop + if not exists ( + select true + from app.allowed_extensions + where + name = require + ) then + raise exception '`requires` in the control file can''t have `%` in it', require; + end if; + end loop; + + insert into app.packages(handle, partial_name, control_description, control_relocatable, control_requires, default_version_struct) + values (account.handle, package_name, package_description, relocatable, requires, app.text_to_semver(default_version)) + on conflict on constraint packages_handle_partial_name_key + do update + set control_description = excluded.control_description, + control_relocatable = excluded.control_relocatable, + control_requires = excluded.control_requires, + default_version_struct = excluded.default_version_struct; +end; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205051816_add_default_version_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205051816_add_default_version_80.snap new file mode 100644 index 000000000..b912fb710 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205051816_add_default_version_80.snap @@ -0,0 +1,111 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231205051816_add_default_version.sql +--- +alter table app.packages + add column default_version_struct app.semver + not null + default app.text_to_semver('0.0.0'), + add column default_version text + generated always as (app.semver_to_text( + default_version_struct + )) stored; + +update app.packages +set default_version_struct = app.text_to_semver(pp.latest_version) +from public.packages as pp +where + packages.id = pp.id; + +alter table app.packages + alter column default_version_struct drop default; + +create or replace view public.packages +as select + pa.id, + pa.package_name, + pa.handle, + pa.partial_name, + newest_ver.version as latest_version, + newest_ver.description_md, + pa.control_description, + pa.control_requires, + pa.created_at, + pa.default_version +from + app.packages as pa, + lateral ( + select + * + from + app.package_versions as pv + where + pv.package_id = pa.id + order by pv.version_struct desc + limit 1 + ) + as newest_ver; + +grant INSERT (partial_name, +handle, +control_description, +control_relocatable, +control_requires, +default_version_struct) +on table app.packages +to authenticated; + +grant UPDATE (control_description, +control_relocatable, +control_requires, +default_version_struct) +on table app.packages +to authenticated; + +drop function public.publish_package(app.valid_name, varchar, boolean, text[]); + +create or replace function +public.publish_package( + package_name app.valid_name, + package_description varchar(1000), + relocatable boolean default false, + requires text[] default '{}', + default_version text default null +) +returns void +language plpgsql +as $function$ +declare + account app.accounts = account from app.accounts account where id = auth.uid(); + require text; +begin + if account.handle is null then + raise exception 'user not logged in'; + end if; + + if default_version is null then + raise exception 'default_version is required. If you are on `dbdev` CLI version 0.1.5 or older upgrade to the latest version.'; + end if; + + foreach require in array requires + loop + if not exists ( + select true + from app.allowed_extensions + where + name = require + ) then + raise exception '`requires` in the control file can''t have `%` in it', require; + end if; + end loop; + + insert into app.packages(handle, partial_name, control_description, control_relocatable, control_requires, default_version_struct) + values (account.handle, package_name, package_description, relocatable, requires, app.text_to_semver(default_version)) + on conflict on constraint packages_handle_partial_name_key + do update + set control_description = excluded.control_description, + control_relocatable = excluded.control_relocatable, + control_requires = excluded.control_requires, + default_version_struct = excluded.default_version_struct; +end; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205101809_dbdev_supports_default_version_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205101809_dbdev_supports_default_version_100.snap new file mode 100644 index 000000000..9265da409 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205101809_dbdev_supports_default_version_100.snap @@ -0,0 +1,352 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231205101809_dbdev_supports_default_version.sql +--- +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_name = 'supabase-dbdev' + ), + (0, 0, 4), + ' + +create schema dbdev; + +-- base_url and api_key have been added as arguments with default values to help test locally +create or replace function dbdev.install( + package_name text, + base_url text default ''https://api.database.dev/rest/v1/'', + api_key text default ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyYndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzIsImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJrzM0AQKsu_5k134s'' +) + returns bool + language plpgsql +as $$ +declare + http_ext_schema regnamespace = extnamespace::regnamespace from pg_catalog.pg_extension where extname = ''http'' limit 1; + pgtle_is_available bool = true from pg_catalog.pg_extension where extname = ''pg_tle'' limit 1; + -- HTTP respones + rec jsonb; + status int; + contents json; + + -- Install Record + rec_sql text; + rec_ver text; + rec_from_ver text; + rec_to_ver text; + rec_package_name text; + rec_description text; + rec_requires text[]; + rec_default_ver text; +begin + + if http_ext_schema is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the http extension and it is not available''); + end if; + + if pgtle_is_available is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the pgtle extension and it is not available''); + end if; + + ------------------- + -- Base Versions -- + ------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_versions?select=package_name,version,sql,control_description,control_requires&limit=50&package_name=eq.%s'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading versions from dbdev''); + end if; + + if contents is null or json_typeof(contents) <> ''array'' or json_array_length(contents) = 0 then + raise exception using errcode=''22000'', message=format(''No versions found for package named %s'', package_name); + end if; + + for rec_package_name, rec_ver, rec_sql, rec_description, rec_requires in select + (r ->> ''package_name''), + (r ->> ''version''), + (r ->> ''sql''), + (r ->> ''control_description''), + array(select json_array_elements_text((r -> ''control_requires''))) + from + json_array_elements(contents) as r + loop + + -- Install the primary version + if not exists ( + select true + from pgtle.available_extensions() + where + name = rec_package_name + ) then + perform pgtle.install_extension(rec_package_name, rec_ver, rec_description, rec_sql, rec_requires); + end if; + + -- Install other available versions + if not exists ( + select true + from pgtle.available_extension_versions() + where + name = rec_package_name + and version = rec_ver + ) then + perform pgtle.install_extension_version_sql(rec_package_name, rec_ver, rec_sql); + end if; + + end loop; + + ---------------------- + -- Upgrade Versions -- + ---------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_upgrades?select=package_name,from_version,to_version,sql&limit=50&package_name=eq.%s'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading upgrade paths from dbdev''); + end if; + + if json_typeof(contents) <> ''array'' then + raise exception using errcode=''22000'', message=format(''Invalid response from dbdev upgrade paths''); + end if; + + for rec_package_name, rec_from_ver, rec_to_ver, rec_sql in select + (r ->> ''package_name''), + (r ->> ''from_version''), + (r ->> ''to_version''), + (r ->> ''sql'') + from + json_array_elements(contents) as r + loop + + if not exists ( + select true + from pgtle.extension_update_paths(rec_package_name) + where + source = rec_from_ver + and target = rec_to_ver + and path is not null + ) then + perform pgtle.install_update_path(rec_package_name, rec_from_ver, rec_to_ver, rec_sql); + end if; + end loop; + + ------------------------- + -- Set Default Version -- + ------------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackages?select=package_name,default_version&limit=1&package_name=eq.%s'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading packages from dbdev''); + end if; + + if contents is null or json_typeof(contents) <> ''array'' or json_array_length(contents) = 0 then + raise exception using errcode=''22000'', message=format(''No package named %s found'', package_name); + end if; + + for rec_package_name, rec_default_ver in select + (r ->> ''package_name''), + (r ->> ''default_version'') + from + json_array_elements(contents) as r + loop + + if rec_default_ver is not null then + perform pgtle.set_default_version(rec_package_name, rec_default_ver); + else + raise notice using errcode=''22000'', message=format(''DBDEV INFO: missing default version''); + end if; + + end loop; + + -------------------------- + -- Send Download Notice -- + -------------------------- + -- Notifies dbdev that a package has been downloaded and records IP + user agent so we can compute unique download counts + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''POST'', + format( + ''%srpc/register_download'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header, + (''x-client-info'', ''dbdev/0.0.4'')::http_header + ], + ''application/json'', + json_build_object(''package_name'', $stmt$ || pg_catalog.quote_literal($1) || $stmt$)::text + ) + ) x + limit 1; $stmt$ + into rec; + + return true; +end; +$$; + +', + ' +# dbdev + +dbdev is the SQL client for database.new and is the primary way end users interact with the package (pglet) registry. + +dbdev can be used to load packages from the registry. For example: + +```sql +-- Load the package from the package index +select dbdev.install(''olirice-index_advisor''); +``` +Where `olirice` is the handle of the author and `index_advisor` is the name of the pglet. + +Once installed, pglets are visible in PostgreSQL as extensions. At that point they can be enabled with standard Postgres commands i.e. the `create extension` + +To improve reproducibility, we recommend __always__ specifying the package version in your `create extension` statements. + +For example: +```sql +-- Enable the extension +create extension "olirice-index_advisor" + schema ''public'' + version ''0.1.0''; +``` + +Which creates all tables/indexes/functions/etc specified by the extension. + +## How to Install + +The in-database SQL client for the package registry is named `dbdev`. You can bootstrap the client with: + +```sql +/*--------------------- +---- install dbdev ---- +---------------------- +Requires: + - pg_tle: https://github.com/aws/pg_tle + - pgsql-http: https://github.com/pramsey/pgsql-http +*/ +create extension if not exists http with schema extensions; +create extension if not exists pg_tle; +drop extension if exists "supabase-dbdev"; +select pgtle.uninstall_extension_if_exists(''supabase-dbdev''); +select + pgtle.install_extension( + ''supabase-dbdev'', + resp.contents ->> ''version'', + ''PostgreSQL package manager'', + resp.contents ->> ''sql'' + ) +from http( + ( + ''GET'', + ''https://api.database.dev/rest/v1/'' + || ''package_versions?select=sql,version'' + || ''&package_name=eq.supabase-dbdev'' + || ''&order=version.desc'' + || ''&limit=1'', + array[ + ( + ''apiKey'', + ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJp'' + || ''c3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyY'' + || ''ndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzI'' + || ''sImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJ'' + || ''rzM0AQKsu_5k134s'' + )::http_header + ], + null, + null + ) +) x, +lateral ( + select + ((row_to_json(x) -> ''content'') #>> ''{}'')::json -> 0 +) resp(contents); +create extension "supabase-dbdev"; +select dbdev.install(''supabase-dbdev''); +drop extension if exists "supabase-dbdev"; +create extension "supabase-dbdev"; +``` + +With the client ready, search for packages on [database.dev](database.dev) and install them with + +```sql +select dbdev.install(''handle-package_name''); +create extension "handle-package_name" + schema ''public'' + version ''1.2.3''; +``` +' + ); + +update app.packages +set default_version_struct = app.text_to_semver('0.0.4') +where + package_name = 'supabase-dbdev'; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205101809_dbdev_supports_default_version_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205101809_dbdev_supports_default_version_80.snap new file mode 100644 index 000000000..c70986b61 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231205101809_dbdev_supports_default_version_80.snap @@ -0,0 +1,357 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231205101809_dbdev_supports_default_version.sql +--- +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_name = 'supabase-dbdev' + ), + (0, 0, 4), + ' + +create schema dbdev; + +-- base_url and api_key have been added as arguments with default values to help test locally +create or replace function dbdev.install( + package_name text, + base_url text default ''https://api.database.dev/rest/v1/'', + api_key text default ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyYndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzIsImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJrzM0AQKsu_5k134s'' +) + returns bool + language plpgsql +as $$ +declare + http_ext_schema regnamespace = extnamespace::regnamespace from pg_catalog.pg_extension where extname = ''http'' limit 1; + pgtle_is_available bool = true from pg_catalog.pg_extension where extname = ''pg_tle'' limit 1; + -- HTTP respones + rec jsonb; + status int; + contents json; + + -- Install Record + rec_sql text; + rec_ver text; + rec_from_ver text; + rec_to_ver text; + rec_package_name text; + rec_description text; + rec_requires text[]; + rec_default_ver text; +begin + + if http_ext_schema is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the http extension and it is not available''); + end if; + + if pgtle_is_available is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the pgtle extension and it is not available''); + end if; + + ------------------- + -- Base Versions -- + ------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_versions?select=package_name,version,sql,control_description,control_requires&limit=50&package_name=eq.%s'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading versions from dbdev''); + end if; + + if contents is null or json_typeof(contents) <> ''array'' or json_array_length(contents) = 0 then + raise exception using errcode=''22000'', message=format(''No versions found for package named %s'', package_name); + end if; + + for rec_package_name, rec_ver, rec_sql, rec_description, rec_requires in select + (r ->> ''package_name''), + (r ->> ''version''), + (r ->> ''sql''), + (r ->> ''control_description''), + array(select json_array_elements_text((r -> ''control_requires''))) + from + json_array_elements(contents) as r + loop + + -- Install the primary version + if not exists ( + select true + from pgtle.available_extensions() + where + name = rec_package_name + ) then + perform pgtle.install_extension(rec_package_name, rec_ver, rec_description, rec_sql, rec_requires); + end if; + + -- Install other available versions + if not exists ( + select true + from pgtle.available_extension_versions() + where + name = rec_package_name + and version = rec_ver + ) then + perform pgtle.install_extension_version_sql(rec_package_name, rec_ver, rec_sql); + end if; + + end loop; + + ---------------------- + -- Upgrade Versions -- + ---------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_upgrades?select=package_name,from_version,to_version,sql&limit=50&package_name=eq.%s'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading upgrade paths from dbdev''); + end if; + + if json_typeof(contents) <> ''array'' then + raise exception using errcode=''22000'', message=format(''Invalid response from dbdev upgrade paths''); + end if; + + for rec_package_name, rec_from_ver, rec_to_ver, rec_sql in select + (r ->> ''package_name''), + (r ->> ''from_version''), + (r ->> ''to_version''), + (r ->> ''sql'') + from + json_array_elements(contents) as r + loop + + if not exists ( + select true + from pgtle.extension_update_paths(rec_package_name) + where + source = rec_from_ver + and target = rec_to_ver + and path is not null + ) then + perform pgtle.install_update_path(rec_package_name, rec_from_ver, rec_to_ver, rec_sql); + end if; + end loop; + + ------------------------- + -- Set Default Version -- + ------------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackages?select=package_name,default_version&limit=1&package_name=eq.%s'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal($1) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading packages from dbdev''); + end if; + + if contents is null or json_typeof(contents) <> ''array'' or json_array_length(contents) = 0 then + raise exception using errcode=''22000'', message=format(''No package named %s found'', package_name); + end if; + + for rec_package_name, rec_default_ver in select + (r ->> ''package_name''), + (r ->> ''default_version'') + from + json_array_elements(contents) as r + loop + + if rec_default_ver is not null then + perform pgtle.set_default_version(rec_package_name, rec_default_ver); + else + raise notice using errcode=''22000'', message=format(''DBDEV INFO: missing default version''); + end if; + + end loop; + + -------------------------- + -- Send Download Notice -- + -------------------------- + -- Notifies dbdev that a package has been downloaded and records IP + user agent so we can compute unique download counts + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''POST'', + format( + ''%srpc/register_download'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header, + (''x-client-info'', ''dbdev/0.0.4'')::http_header + ], + ''application/json'', + json_build_object(''package_name'', $stmt$ || pg_catalog.quote_literal($1) || $stmt$)::text + ) + ) x + limit 1; $stmt$ + into rec; + + return true; +end; +$$; + +', + ' +# dbdev + +dbdev is the SQL client for database.new and is the primary way end users interact with the package (pglet) registry. + +dbdev can be used to load packages from the registry. For example: + +```sql +-- Load the package from the package index +select dbdev.install(''olirice-index_advisor''); +``` +Where `olirice` is the handle of the author and `index_advisor` is the name of the pglet. + +Once installed, pglets are visible in PostgreSQL as extensions. At that point they can be enabled with standard Postgres commands i.e. the `create extension` + +To improve reproducibility, we recommend __always__ specifying the package version in your `create extension` statements. + +For example: +```sql +-- Enable the extension +create extension "olirice-index_advisor" + schema ''public'' + version ''0.1.0''; +``` + +Which creates all tables/indexes/functions/etc specified by the extension. + +## How to Install + +The in-database SQL client for the package registry is named `dbdev`. You can bootstrap the client with: + +```sql +/*--------------------- +---- install dbdev ---- +---------------------- +Requires: + - pg_tle: https://github.com/aws/pg_tle + - pgsql-http: https://github.com/pramsey/pgsql-http +*/ +create extension if not exists http with schema extensions; +create extension if not exists pg_tle; +drop extension if exists "supabase-dbdev"; +select pgtle.uninstall_extension_if_exists(''supabase-dbdev''); +select + pgtle.install_extension( + ''supabase-dbdev'', + resp.contents ->> ''version'', + ''PostgreSQL package manager'', + resp.contents ->> ''sql'' + ) +from http( + ( + ''GET'', + ''https://api.database.dev/rest/v1/'' + || ''package_versions?select=sql,version'' + || ''&package_name=eq.supabase-dbdev'' + || ''&order=version.desc'' + || ''&limit=1'', + array[ + ( + ''apiKey'', + ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJp'' + || ''c3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyY'' + || ''ndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzI'' + || ''sImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJ'' + || ''rzM0AQKsu_5k134s'' + )::http_header + ], + null, + null + ) +) x, +lateral ( + select + ((row_to_json(x) -> ''content'') #>> ''{}'')::json -> 0 +) resp(contents); +create extension "supabase-dbdev"; +select dbdev.install(''supabase-dbdev''); +drop extension if exists "supabase-dbdev"; +create extension "supabase-dbdev"; +``` + +With the client ready, search for packages on [database.dev](database.dev) and install them with + +```sql +select dbdev.install(''handle-package_name''); +create extension "handle-package_name" + schema ''public'' + version ''1.2.3''; +``` +' + ); + +update app.packages +set default_version_struct = app.text_to_semver('0.0.4') +where + package_name = 'supabase-dbdev'; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207071422_new_package_name_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207071422_new_package_name_100.snap new file mode 100644 index 000000000..f5a8786d5 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207071422_new_package_name_100.snap @@ -0,0 +1,88 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231207071422_new_package_name.sql +--- +create or replace function app.to_package_name(handle app.valid_name, partial_name app.valid_name) +returns text +language sql +immutable +as $function$ +select format('%s@%s', $1, $2) +$function$; + +alter table app.packages + add column package_alias text null; + +update app.packages set package_alias = format('%s@%s', handle, partial_name); + +create or replace view public.packages +as select + pa.id, + pa.package_name, + pa.handle, + pa.partial_name, + newest_ver.version as latest_version, + newest_ver.description_md, + pa.control_description, + pa.control_requires, + pa.created_at, + pa.default_version, + pa.package_alias +from + app.packages as pa, + lateral ( + select + * + from + app.package_versions as pv + where + pv.package_id = pa.id + order by pv.version_struct desc + limit 1 + ) + as newest_ver; + +create or replace view public.package_versions +as select + pv.id, + pv.package_id, + pa.package_name, + pv.version, + pv.sql, + pv.description_md, + pa.control_description, + pa.control_requires, + pv.created_at, + pa.package_alias +from + app.packages as pa + inner join + app.package_versions as pv + on pa.id = pv.package_id; + +create or replace view public.package_upgrades +as select + pu.id, + pu.package_id, + pa.package_name, + pu.from_version, + pu.to_version, + pu.sql, + pu.created_at, + pa.package_alias +from + app.packages as pa + inner join + app.package_upgrades as pu + on pa.id = pu.package_id; + +create or replace function public.register_download(package_name text) +returns void +language sql +security definer +as $function$ +insert into app.downloads(package_id) + select id + from app.packages ap + where ap.package_name = $1 or ap.package_alias = $1 +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207071422_new_package_name_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207071422_new_package_name_80.snap new file mode 100644 index 000000000..47fec4023 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207071422_new_package_name_80.snap @@ -0,0 +1,92 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231207071422_new_package_name.sql +--- +create or replace function +app.to_package_name( + handle app.valid_name, + partial_name app.valid_name +) +returns text +language sql +immutable +as $function$ +select format('%s@%s', $1, $2) +$function$; + +alter table app.packages + add column package_alias text null; + +update app.packages set package_alias = format('%s@%s', handle, partial_name); + +create or replace view public.packages +as select + pa.id, + pa.package_name, + pa.handle, + pa.partial_name, + newest_ver.version as latest_version, + newest_ver.description_md, + pa.control_description, + pa.control_requires, + pa.created_at, + pa.default_version, + pa.package_alias +from + app.packages as pa, + lateral ( + select + * + from + app.package_versions as pv + where + pv.package_id = pa.id + order by pv.version_struct desc + limit 1 + ) + as newest_ver; + +create or replace view public.package_versions +as select + pv.id, + pv.package_id, + pa.package_name, + pv.version, + pv.sql, + pv.description_md, + pa.control_description, + pa.control_requires, + pv.created_at, + pa.package_alias +from + app.packages as pa + inner join + app.package_versions as pv + on pa.id = pv.package_id; + +create or replace view public.package_upgrades +as select + pu.id, + pu.package_id, + pa.package_name, + pu.from_version, + pu.to_version, + pu.sql, + pu.created_at, + pa.package_alias +from + app.packages as pa + inner join + app.package_upgrades as pu + on pa.id = pu.package_id; + +create or replace function public.register_download(package_name text) +returns void +language sql +security definer +as $function$ +insert into app.downloads(package_id) + select id + from app.packages ap + where ap.package_name = $1 or ap.package_alias = $1 +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207073048_dbdev_supports_new_package_names_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207073048_dbdev_supports_new_package_names_100.snap new file mode 100644 index 000000000..0b9b8471d --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207073048_dbdev_supports_new_package_names_100.snap @@ -0,0 +1,354 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231207073048_dbdev_supports_new_package_names.sql +--- +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_alias = 'supabase@dbdev' + ), + (0, 0, 5), + ' + +create schema dbdev; + +-- base_url and api_key have been added as arguments with default values to help test locally +create or replace function dbdev.install( + package_name text, + base_url text default ''https://api.database.dev/rest/v1/'', + api_key text default ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyYndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzIsImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJrzM0AQKsu_5k134s'' +) + returns bool + language plpgsql +as $$ +declare + http_ext_schema regnamespace = extnamespace::regnamespace from pg_catalog.pg_extension where extname = ''http'' limit 1; + pgtle_is_available bool = true from pg_catalog.pg_extension where extname = ''pg_tle'' limit 1; + -- HTTP respones + rec jsonb; + status int; + contents json; + + -- Install Record + rec_sql text; + rec_ver text; + rec_from_ver text; + rec_to_ver text; + rec_description text; + rec_requires text[]; + rec_default_ver text; +begin + + if http_ext_schema is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the http extension and it is not available''); + end if; + + if pgtle_is_available is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the pgtle extension and it is not available''); + end if; + + ------------------- + -- Base Versions -- + ------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_versions?select=version,sql,control_description,control_requires&limit=50&or=(package_name.eq.%s,package_alias.eq.%s)'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading versions from dbdev''); + end if; + + if contents is null or json_typeof(contents) <> ''array'' or json_array_length(contents) = 0 then + raise exception using errcode=''22000'', message=format(''No versions found for package named %s'', package_name); + end if; + + for rec_ver, rec_sql, rec_description, rec_requires in select + (r ->> ''version''), + (r ->> ''sql''), + (r ->> ''control_description''), + array(select json_array_elements_text((r -> ''control_requires''))) + from + json_array_elements(contents) as r + loop + + -- Install the primary version + if not exists ( + select true + from pgtle.available_extensions() + where + name = package_name + ) then + perform pgtle.install_extension(package_name, rec_ver, rec_description, rec_sql, rec_requires); + end if; + + -- Install other available versions + if not exists ( + select true + from pgtle.available_extension_versions() + where + name = package_name + and version = rec_ver + ) then + perform pgtle.install_extension_version_sql(package_name, rec_ver, rec_sql); + end if; + + end loop; + + ---------------------- + -- Upgrade Versions -- + ---------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_upgrades?select=from_version,to_version,sql&limit=50&or=(package_name.eq.%s,package_alias.eq.%s)'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading upgrade paths from dbdev''); + end if; + + if json_typeof(contents) <> ''array'' then + raise exception using errcode=''22000'', message=format(''Invalid response from dbdev upgrade paths''); + end if; + + for rec_from_ver, rec_to_ver, rec_sql in select + (r ->> ''from_version''), + (r ->> ''to_version''), + (r ->> ''sql'') + from + json_array_elements(contents) as r + loop + + if not exists ( + select true + from pgtle.extension_update_paths(package_name) + where + source = rec_from_ver + and target = rec_to_ver + and path is not null + ) then + perform pgtle.install_update_path(package_name, rec_from_ver, rec_to_ver, rec_sql); + end if; + end loop; + + ------------------------- + -- Set Default Version -- + ------------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackages?select=default_version&limit=1&or=(package_name.eq.%s,package_alias.eq.%s)'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading packages from dbdev''); + end if; + + if contents is null or json_typeof(contents) <> ''array'' or json_array_length(contents) = 0 then + raise exception using errcode=''22000'', message=format(''No package named %s found'', package_name); + end if; + + for rec_default_ver in select + (r ->> ''default_version'') + from + json_array_elements(contents) as r + loop + + if rec_default_ver is not null then + perform pgtle.set_default_version(package_name, rec_default_ver); + else + raise notice using errcode=''22000'', message=format(''DBDEV INFO: missing default version''); + end if; + + end loop; + + -------------------------- + -- Send Download Notice -- + -------------------------- + -- Notifies dbdev that a package has been downloaded and records IP + user agent so we can compute unique download counts + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''POST'', + format( + ''%srpc/register_download'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header, + (''x-client-info'', ''dbdev/0.0.5'')::http_header + ], + ''application/json'', + json_build_object(''package_name'', $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$)::text + ) + ) x + limit 1; $stmt$ + into rec; + + return true; +end; +$$; + +', + ' +# dbdev + +dbdev is the SQL client for database.new and is the primary way end users interact with the package registry. + +dbdev can be used to load packages from the registry. For example: + +```sql +-- Load the package from the package index +select dbdev.install(''olirice@index_advisor''); +``` +Where `olirice` is the handle of the author and `index_advisor` is the name of the package. + +Once installed, packages are visible in PostgreSQL as extensions. At that point they can be enabled with standard Postgres commands i.e. the `create extension` + +To improve reproducibility, we recommend __always__ specifying the package version in your `create extension` statements. + +For example: +```sql +-- Enable the extension +create extension "olirice@index_advisor" + schema ''public'' + version ''0.1.0''; +``` + +Which creates all tables/indexes/functions/etc specified by the extension. + +## How to Install + +The in-database SQL client for the package registry is named `dbdev`. You can bootstrap the client with: + +```sql +/*--------------------- +---- install dbdev ---- +---------------------- +Requires: + - pg_tle: https://github.com/aws/pg_tle + - pgsql-http: https://github.com/pramsey/pgsql-http +*/ +create extension if not exists http with schema extensions; +create extension if not exists pg_tle; +-- drop dbdev with older naming scheme if present +drop extension if exists "supabase-dbdev"; +select pgtle.uninstall_extension_if_exists(''supabase-dbdev''); +drop extension if exists "supabase@dbdev"; +select pgtle.uninstall_extension_if_exists(''supabase@dbdev''); +select + pgtle.install_extension( + ''supabase@dbdev'', + resp.contents ->> ''version'', + ''PostgreSQL package manager'', + resp.contents ->> ''sql'' + ) +from http( + ( + ''GET'', + ''https://api.database.dev/rest/v1/'' + || ''package_versions?select=sql,version'' + || ''&package_alias=eq.supabase@dbdev'' + || ''&order=version.desc'' + || ''&limit=1'', + array[ + ( + ''apiKey'', + ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJp'' + || ''c3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyY'' + || ''ndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzI'' + || ''sImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJ'' + || ''rzM0AQKsu_5k134s'' + )::http_header + ], + null, + null + ) +) x, +lateral ( + select + ((row_to_json(x) -> ''content'') #>> ''{}'')::json -> 0 +) resp(contents); +create extension "supabase@dbdev"; +select dbdev.install(''supabase@dbdev''); +drop extension if exists "supabase@dbdev"; +create extension "supabase@dbdev"; +``` + +With the client ready, search for packages on [database.dev](database.dev) and install them with + +```sql +select dbdev.install(''handle@package_name''); +create extension "handle@package_name" + schema ''public'' + version ''1.2.3''; +``` +' + ); + +update app.packages +set default_version_struct = app.text_to_semver('0.0.5') +where + package_alias = 'supabase@dbdev'; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207073048_dbdev_supports_new_package_names_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207073048_dbdev_supports_new_package_names_80.snap new file mode 100644 index 000000000..3dc2a4986 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207073048_dbdev_supports_new_package_names_80.snap @@ -0,0 +1,359 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231207073048_dbdev_supports_new_package_names.sql +--- +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_alias = 'supabase@dbdev' + ), + (0, 0, 5), + ' + +create schema dbdev; + +-- base_url and api_key have been added as arguments with default values to help test locally +create or replace function dbdev.install( + package_name text, + base_url text default ''https://api.database.dev/rest/v1/'', + api_key text default ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyYndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzIsImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJrzM0AQKsu_5k134s'' +) + returns bool + language plpgsql +as $$ +declare + http_ext_schema regnamespace = extnamespace::regnamespace from pg_catalog.pg_extension where extname = ''http'' limit 1; + pgtle_is_available bool = true from pg_catalog.pg_extension where extname = ''pg_tle'' limit 1; + -- HTTP respones + rec jsonb; + status int; + contents json; + + -- Install Record + rec_sql text; + rec_ver text; + rec_from_ver text; + rec_to_ver text; + rec_description text; + rec_requires text[]; + rec_default_ver text; +begin + + if http_ext_schema is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the http extension and it is not available''); + end if; + + if pgtle_is_available is null then + raise exception using errcode=''22000'', message=format(''dbdev requires the pgtle extension and it is not available''); + end if; + + ------------------- + -- Base Versions -- + ------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_versions?select=version,sql,control_description,control_requires&limit=50&or=(package_name.eq.%s,package_alias.eq.%s)'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading versions from dbdev''); + end if; + + if contents is null or json_typeof(contents) <> ''array'' or json_array_length(contents) = 0 then + raise exception using errcode=''22000'', message=format(''No versions found for package named %s'', package_name); + end if; + + for rec_ver, rec_sql, rec_description, rec_requires in select + (r ->> ''version''), + (r ->> ''sql''), + (r ->> ''control_description''), + array(select json_array_elements_text((r -> ''control_requires''))) + from + json_array_elements(contents) as r + loop + + -- Install the primary version + if not exists ( + select true + from pgtle.available_extensions() + where + name = package_name + ) then + perform pgtle.install_extension(package_name, rec_ver, rec_description, rec_sql, rec_requires); + end if; + + -- Install other available versions + if not exists ( + select true + from pgtle.available_extension_versions() + where + name = package_name + and version = rec_ver + ) then + perform pgtle.install_extension_version_sql(package_name, rec_ver, rec_sql); + end if; + + end loop; + + ---------------------- + -- Upgrade Versions -- + ---------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackage_upgrades?select=from_version,to_version,sql&limit=50&or=(package_name.eq.%s,package_alias.eq.%s)'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading upgrade paths from dbdev''); + end if; + + if json_typeof(contents) <> ''array'' then + raise exception using errcode=''22000'', message=format(''Invalid response from dbdev upgrade paths''); + end if; + + for rec_from_ver, rec_to_ver, rec_sql in select + (r ->> ''from_version''), + (r ->> ''to_version''), + (r ->> ''sql'') + from + json_array_elements(contents) as r + loop + + if not exists ( + select true + from pgtle.extension_update_paths(package_name) + where + source = rec_from_ver + and target = rec_to_ver + and path is not null + ) then + perform pgtle.install_update_path(package_name, rec_from_ver, rec_to_ver, rec_sql); + end if; + end loop; + + ------------------------- + -- Set Default Version -- + ------------------------- + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''GET'', + format( + ''%spackages?select=default_version&limit=1&or=(package_name.eq.%s,package_alias.eq.%s)'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$, + $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header + ], + null, + null + ) + ) x + limit 1; $stmt$ + into rec; + + status = (rec ->> ''status'')::int; + contents = to_json(rec ->> ''content'') #>> ''{}''; + + if status <> 200 then + raise notice using errcode=''22000'', message=format(''DBDEV INFO: %s'', contents); + raise exception using errcode=''22000'', message=format(''Non-200 response code while loading packages from dbdev''); + end if; + + if contents is null or json_typeof(contents) <> ''array'' or json_array_length(contents) = 0 then + raise exception using errcode=''22000'', message=format(''No package named %s found'', package_name); + end if; + + for rec_default_ver in select + (r ->> ''default_version'') + from + json_array_elements(contents) as r + loop + + if rec_default_ver is not null then + perform pgtle.set_default_version(package_name, rec_default_ver); + else + raise notice using errcode=''22000'', message=format(''DBDEV INFO: missing default version''); + end if; + + end loop; + + -------------------------- + -- Send Download Notice -- + -------------------------- + -- Notifies dbdev that a package has been downloaded and records IP + user agent so we can compute unique download counts + execute $stmt$select row_to_json(x) + from $stmt$ || pg_catalog.quote_ident(http_ext_schema::text) || $stmt$.http( + ( + ''POST'', + format( + ''%srpc/register_download'', + $stmt$ || pg_catalog.quote_literal(base_url) || $stmt$ + ), + array[ + (''apiKey'', $stmt$ || pg_catalog.quote_literal(api_key) || $stmt$)::http_header, + (''x-client-info'', ''dbdev/0.0.5'')::http_header + ], + ''application/json'', + json_build_object(''package_name'', $stmt$ || pg_catalog.quote_literal(package_name) || $stmt$)::text + ) + ) x + limit 1; $stmt$ + into rec; + + return true; +end; +$$; + +', + ' +# dbdev + +dbdev is the SQL client for database.new and is the primary way end users interact with the package registry. + +dbdev can be used to load packages from the registry. For example: + +```sql +-- Load the package from the package index +select dbdev.install(''olirice@index_advisor''); +``` +Where `olirice` is the handle of the author and `index_advisor` is the name of the package. + +Once installed, packages are visible in PostgreSQL as extensions. At that point they can be enabled with standard Postgres commands i.e. the `create extension` + +To improve reproducibility, we recommend __always__ specifying the package version in your `create extension` statements. + +For example: +```sql +-- Enable the extension +create extension "olirice@index_advisor" + schema ''public'' + version ''0.1.0''; +``` + +Which creates all tables/indexes/functions/etc specified by the extension. + +## How to Install + +The in-database SQL client for the package registry is named `dbdev`. You can bootstrap the client with: + +```sql +/*--------------------- +---- install dbdev ---- +---------------------- +Requires: + - pg_tle: https://github.com/aws/pg_tle + - pgsql-http: https://github.com/pramsey/pgsql-http +*/ +create extension if not exists http with schema extensions; +create extension if not exists pg_tle; +-- drop dbdev with older naming scheme if present +drop extension if exists "supabase-dbdev"; +select pgtle.uninstall_extension_if_exists(''supabase-dbdev''); +drop extension if exists "supabase@dbdev"; +select pgtle.uninstall_extension_if_exists(''supabase@dbdev''); +select + pgtle.install_extension( + ''supabase@dbdev'', + resp.contents ->> ''version'', + ''PostgreSQL package manager'', + resp.contents ->> ''sql'' + ) +from http( + ( + ''GET'', + ''https://api.database.dev/rest/v1/'' + || ''package_versions?select=sql,version'' + || ''&package_alias=eq.supabase@dbdev'' + || ''&order=version.desc'' + || ''&limit=1'', + array[ + ( + ''apiKey'', + ''eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJp'' + || ''c3MiOiJzdXBhYmFzZSIsInJlZiI6InhtdXB0cHBsZnZpaWZyY'' + || ''ndtbXR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2ODAxMDczNzI'' + || ''sImV4cCI6MTk5NTY4MzM3Mn0.z2CN0mvO2No8wSi46Gw59DFGCTJ'' + || ''rzM0AQKsu_5k134s'' + )::http_header + ], + null, + null + ) +) x, +lateral ( + select + ((row_to_json(x) -> ''content'') #>> ''{}'')::json -> 0 +) resp(contents); +create extension "supabase@dbdev"; +select dbdev.install(''supabase@dbdev''); +drop extension if exists "supabase@dbdev"; +create extension "supabase@dbdev"; +``` + +With the client ready, search for packages on [database.dev](database.dev) and install them with + +```sql +select dbdev.install(''handle@package_name''); +create extension "handle@package_name" + schema ''public'' + version ''1.2.3''; +``` +' + ); + +update app.packages +set default_version_struct = app.text_to_semver('0.0.5') +where + package_alias = 'supabase@dbdev'; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207111703_langchain@embedding_search-1.1.1_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207111703_langchain@embedding_search-1.1.1_100.snap new file mode 100644 index 000000000..c16c8ce3b --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207111703_langchain@embedding_search-1.1.1_100.snap @@ -0,0 +1,178 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231207111703_langchain@embedding_search-1.1.1.sql +--- +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_alias = 'langchain@embedding_search' + ), + (1, 1, 1), + ' +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''vector'' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception ''"langchain@embedding_search" requires "vector"'' + using hint = ''Run "create extension vector" and try again''; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int, + filter jsonb DEFAULT ''{}'' +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + where metadata @> filter + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +', + ' +# embedding_search + +[LangChain](https://js.langchain.com/docs/) is a framework for developing applications powered by language models with a plugable architecture. + +`langchain@embedding_search` uses a Supabase Postgres database as its vector store. + +## Installation + +```sql +select dbdev.install(''langchain@embedding_search''); +create extension if not exists vector; +create extension "langchain@embedding_search" + schema public + version ''1.1.0''; +``` +Note: + +`vector` is a dependency of `langchain@embedding_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +### Standard Usage + +The below example shows how to perform a basic similarity search with Supabase: + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromTexts( + ["Hello world", "Bye bye", "What''s this?"], + [{ id: 2 }, { id: 1 }, { id: 3 }], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + const resultOne = await vectorStore.similaritySearch("Hello world", 1); + + console.log(resultOne); +}; +``` + +### Metadata Filtering + +Given the above `match_documents` Postgres function, you can also pass a filter parameter to only documents with a specific metadata field value. + +```js +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; + +// First, follow set-up instructions at +// https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromTexts( + ["Hello world", "Hello world", "Hello world"], + [{ user_id: 2 }, { user_id: 1 }, { user_id: 3 }], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + const result = await vectorStore.similaritySearch("Hello world", 1, { + user_id: 3, + }); + + console.log(result); +}; +``` + +For more details, checkout the LangChain Supabase integration docs: https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207111703_langchain@embedding_search-1.1.1_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207111703_langchain@embedding_search-1.1.1_80.snap new file mode 100644 index 000000000..fa6a75c06 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207111703_langchain@embedding_search-1.1.1_80.snap @@ -0,0 +1,184 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231207111703_langchain@embedding_search-1.1.1.sql +--- +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_alias = + 'langchain@embedding_search' + ), + (1, 1, 1), + ' +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''vector'' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception ''"langchain@embedding_search" requires "vector"'' + using hint = ''Run "create extension vector" and try again''; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int, + filter jsonb DEFAULT ''{}'' +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + where metadata @> filter + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +', + ' +# embedding_search + +[LangChain](https://js.langchain.com/docs/) is a framework for developing applications powered by language models with a plugable architecture. + +`langchain@embedding_search` uses a Supabase Postgres database as its vector store. + +## Installation + +```sql +select dbdev.install(''langchain@embedding_search''); +create extension if not exists vector; +create extension "langchain@embedding_search" + schema public + version ''1.1.0''; +``` +Note: + +`vector` is a dependency of `langchain@embedding_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +### Standard Usage + +The below example shows how to perform a basic similarity search with Supabase: + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromTexts( + ["Hello world", "Bye bye", "What''s this?"], + [{ id: 2 }, { id: 1 }, { id: 3 }], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + const resultOne = await vectorStore.similaritySearch("Hello world", 1); + + console.log(resultOne); +}; +``` + +### Metadata Filtering + +Given the above `match_documents` Postgres function, you can also pass a filter parameter to only documents with a specific metadata field value. + +```js +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; + +// First, follow set-up instructions at +// https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromTexts( + ["Hello world", "Hello world", "Hello world"], + [{ user_id: 2 }, { user_id: 1 }, { user_id: 3 }], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + const result = await vectorStore.similaritySearch("Hello world", 1, { + user_id: 3, + }); + + console.log(result); +}; +``` + +For more details, checkout the LangChain Supabase integration docs: https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112129_langchain@hybrid_search-1.1.1_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112129_langchain@hybrid_search-1.1.1_100.snap new file mode 100644 index 000000000..2f6dc5d6a --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112129_langchain@hybrid_search-1.1.1_100.snap @@ -0,0 +1,155 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231207112129_langchain@hybrid_search-1.1.1.sql +--- +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_alias = 'langchain@hybrid_search' + ), + (1, 1, 1), + ' +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''vector'' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception ''"langchain@hybrid_search" requires "vector"'' + using hint = ''Run "create extension vector" and try again''; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to similarity search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int, + filter jsonb DEFAULT ''{}'' +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + where metadata @> filter + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +-- Create a function to keyword search for documents +create function kw_match_documents(query_text text, match_count int) +returns table (id bigint, content text, metadata jsonb, similarity real) +as $$ + +begin +return query execute +format('' + select + id, content, metadata, ts_rank(to_tsvector(content), plainto_tsquery($1)) as similarity + from + documents + where + to_tsvector(content) @@ plainto_tsquery($1) + order by + similarity desc + limit $2 +'') +using query_text, match_count; +end; +$$ language plpgsql; + +', + ' +# hybrid_search + +Langchain supports hybrid search with a Supabase Postgres database. The hybrid search combines the postgres pgvector extension (similarity search) and Full-Text Search (keyword search) to retrieve documents. You can add documents via SupabaseVectorStore addDocuments function. SupabaseHybridKeyWordSearch accepts embedding, supabase client, number of results for similarity search, and number of results for keyword search as parameters. The getRelevantDocuments function produces a list of documents that has duplicates removed and is sorted by relevance score. + +## Installation + +```sql +select dbdev.install(''langchain@hybrid_search''); +create extension if not exists vector; +create extension "langchain@hybrid_search" + schema public + version ''1.1.0''; +``` +Note: + +`vector` is a dependency of `langchain@hybrid_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; +import { SupabaseHybridSearch } from "langchain/retrievers/supabase"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const embeddings = new OpenAIEmbeddings(); + + const retriever = new SupabaseHybridSearch(embeddings, { + client, + // Below are the defaults, expecting that you set up your supabase table and functions according to the guide above. Please change if necessary. + similarityK: 2, + keywordK: 2, + tableName: "documents", + similarityQueryName: "match_documents", + keywordQueryName: "kw_match_documents", + }); + + const results = await retriever.getRelevantDocuments("hello bye"); + + console.log(results); +}; +``` + +For more details, checkout the LangChain Supabase Hybrid Search docs: https://js.langchain.com/docs/modules/indexes/retrievers/supabase-hybrid +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112129_langchain@hybrid_search-1.1.1_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112129_langchain@hybrid_search-1.1.1_80.snap new file mode 100644 index 000000000..e20489156 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112129_langchain@hybrid_search-1.1.1_80.snap @@ -0,0 +1,161 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231207112129_langchain@hybrid_search-1.1.1.sql +--- +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_alias = + 'langchain@hybrid_search' + ), + (1, 1, 1), + ' +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + dependencies_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''vector'' + and installed_version is not null + ); + begin + + if not dependencies_exists then + raise + exception ''"langchain@hybrid_search" requires "vector"'' + using hint = ''Run "create extension vector" and try again''; + end if; + end +$$; + +-- Create a table to store your documents +create table documents ( + id bigserial primary key, + content text, -- corresponds to Document.pageContent + metadata jsonb, -- corresponds to Document.metadata + embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed +); + +-- Create a function to similarity search for documents +create function match_documents ( + query_embedding vector(1536), + match_count int, + filter jsonb DEFAULT ''{}'' +) returns table ( + id bigint, + content text, + metadata jsonb, + similarity float +) +language plpgsql +as $$ +#variable_conflict use_column +begin + return query + select + id, + content, + metadata, + 1 - (documents.embedding <=> query_embedding) as similarity + from documents + where metadata @> filter + order by documents.embedding <=> query_embedding + limit match_count; +end; +$$; + +-- Create a function to keyword search for documents +create function kw_match_documents(query_text text, match_count int) +returns table (id bigint, content text, metadata jsonb, similarity real) +as $$ + +begin +return query execute +format('' + select + id, content, metadata, ts_rank(to_tsvector(content), plainto_tsquery($1)) as similarity + from + documents + where + to_tsvector(content) @@ plainto_tsquery($1) + order by + similarity desc + limit $2 +'') +using query_text, match_count; +end; +$$ language plpgsql; + +', + ' +# hybrid_search + +Langchain supports hybrid search with a Supabase Postgres database. The hybrid search combines the postgres pgvector extension (similarity search) and Full-Text Search (keyword search) to retrieve documents. You can add documents via SupabaseVectorStore addDocuments function. SupabaseHybridKeyWordSearch accepts embedding, supabase client, number of results for similarity search, and number of results for keyword search as parameters. The getRelevantDocuments function produces a list of documents that has duplicates removed and is sorted by relevance score. + +## Installation + +```sql +select dbdev.install(''langchain@hybrid_search''); +create extension if not exists vector; +create extension "langchain@hybrid_search" + schema public + version ''1.1.0''; +``` +Note: + +`vector` is a dependency of `langchain@hybrid_search`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + + +Once created, you can access the vector store for search using langchain as shown below: + +```js +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; +import { SupabaseHybridSearch } from "langchain/retrievers/supabase"; + +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +export const run = async () => { + const client = createClient(url, privateKey); + + const embeddings = new OpenAIEmbeddings(); + + const retriever = new SupabaseHybridSearch(embeddings, { + client, + // Below are the defaults, expecting that you set up your supabase table and functions according to the guide above. Please change if necessary. + similarityK: 2, + keywordK: 2, + tableName: "documents", + similarityQueryName: "match_documents", + keywordQueryName: "kw_match_documents", + }); + + const results = await retriever.getRelevantDocuments("hello bye"); + + console.log(results); +}; +``` + +For more details, checkout the LangChain Supabase Hybrid Search docs: https://js.langchain.com/docs/modules/indexes/retrievers/supabase-hybrid +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112942_michelp@adminpack-0.0.2_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112942_michelp@adminpack-0.0.2_100.snap new file mode 100644 index 000000000..750a27864 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112942_michelp@adminpack-0.0.2_100.snap @@ -0,0 +1,1566 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231207112942_michelp@adminpack-0.0.2.sql +--- +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_alias = 'michelp@adminpack' + ), + (0, 0, 2), + ' +-- From: https://github.com/ioguix/pgsql-bloat-estimation + +-- Copyright (c) 2015-2019, Jehan-Guillaume (ioguix) de Rorthais +-- All rights reserved. + +-- Redistribution and use in source and binary forms, with or without +-- modification, are permitted provided that the following conditions are met: + +-- * Redistributions of source code must retain the above copyright notice, this +-- list of conditions and the following disclaimer. + +-- * Redistributions in binary form must reproduce the above copyright notice, +-- this list of conditions and the following disclaimer in the documentation +-- and/or other materials provided with the distribution. + +-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +-- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +-- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +-- DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +-- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +-- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +-- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +-- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +-- OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +-- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-- WARNING: executed with a non-superuser role, the query inspect only index on tables you are granted to read. +-- WARNING: rows with is_na = ''t'' are known to have bad statistics ("name" type is not supported). +-- This query is compatible with PostgreSQL 8.2 and after +CREATE VIEW index_bloat AS SELECT current_database(), nspname AS schemaname, tblname, idxname, bs*(relpages)::bigint AS real_size, + bs*(relpages-est_pages)::bigint AS extra_size, + 100 * (relpages-est_pages)::float / relpages AS extra_pct, + fillfactor, + CASE WHEN relpages > est_pages_ff + THEN bs*(relpages-est_pages_ff) + ELSE 0 + END AS bloat_size, + 100 * (relpages-est_pages_ff)::float / relpages AS bloat_pct, + is_na + -- , 100-(pst).avg_leaf_density AS pst_avg_bloat, est_pages, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples, relpages -- (DEBUG INFO) +FROM ( + SELECT coalesce(1 + + ceil(reltuples/floor((bs-pageopqdata-pagehdr)/(4+nulldatahdrwidth)::float)), 0 -- ItemIdData size + computed avg size of a tuple (nulldatahdrwidth) + ) AS est_pages, + coalesce(1 + + ceil(reltuples/floor((bs-pageopqdata-pagehdr)*fillfactor/(100*(4+nulldatahdrwidth)::float))), 0 + ) AS est_pages_ff, + bs, nspname, tblname, idxname, relpages, fillfactor, is_na + -- , pgstatindex(idxoid) AS pst, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples -- (DEBUG INFO) + FROM ( + SELECT maxalign, bs, nspname, tblname, idxname, reltuples, relpages, idxoid, fillfactor, + ( index_tuple_hdr_bm + + maxalign - CASE -- Add padding to the index tuple header to align on MAXALIGN + WHEN index_tuple_hdr_bm%maxalign = 0 THEN maxalign + ELSE index_tuple_hdr_bm%maxalign + END + + nulldatawidth + maxalign - CASE -- Add padding to the data to align on MAXALIGN + WHEN nulldatawidth = 0 THEN 0 + WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign + ELSE nulldatawidth::integer%maxalign + END + )::numeric AS nulldatahdrwidth, pagehdr, pageopqdata, is_na + -- , index_tuple_hdr_bm, nulldatawidth -- (DEBUG INFO) + FROM ( + SELECT n.nspname, i.tblname, i.idxname, i.reltuples, i.relpages, + i.idxoid, i.fillfactor, current_setting(''block_size'')::numeric AS bs, + CASE -- MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?) + WHEN version() ~ ''mingw32'' OR version() ~ ''64-bit|x86_64|ppc64|ia64|amd64'' THEN 8 + ELSE 4 + END AS maxalign, + /* per page header, fixed size: 20 for 7.X, 24 for others */ + 24 AS pagehdr, + /* per page btree opaque data */ + 16 AS pageopqdata, + /* per tuple header: add IndexAttributeBitMapData if some cols are null-able */ + CASE WHEN max(coalesce(s.null_frac,0)) = 0 + THEN 8 -- IndexTupleData size + ELSE 8 + (( 32 + 8 - 1 ) / 8) -- IndexTupleData size + IndexAttributeBitMapData size ( max num filed per index + 8 - 1 /8) + END AS index_tuple_hdr_bm, + /* data len: we remove null values save space using it fractionnal part from stats */ + sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) AS nulldatawidth, + max( CASE WHEN i.atttypid = ''pg_catalog.name''::regtype THEN 1 ELSE 0 END ) > 0 AS is_na + FROM ( + SELECT ct.relname AS tblname, ct.relnamespace, ic.idxname, ic.attpos, ic.indkey, ic.indkey[ic.attpos], ic.reltuples, ic.relpages, ic.tbloid, ic.idxoid, ic.fillfactor, + coalesce(a1.attnum, a2.attnum) AS attnum, coalesce(a1.attname, a2.attname) AS attname, coalesce(a1.atttypid, a2.atttypid) AS atttypid, + CASE WHEN a1.attnum IS NULL + THEN ic.idxname + ELSE ct.relname + END AS attrelname + FROM ( + SELECT idxname, reltuples, relpages, tbloid, idxoid, fillfactor, indkey, + pg_catalog.generate_series(1,indnatts) AS attpos + FROM ( + SELECT ci.relname AS idxname, ci.reltuples, ci.relpages, i.indrelid AS tbloid, + i.indexrelid AS idxoid, + coalesce(substring( + array_to_string(ci.reloptions, '' '') + from ''fillfactor=([0-9]+)'')::smallint, 90) AS fillfactor, + i.indnatts, + pg_catalog.string_to_array(pg_catalog.textin( + pg_catalog.int2vectorout(i.indkey)),'' '')::int[] AS indkey + FROM pg_catalog.pg_index i + JOIN pg_catalog.pg_class ci ON ci.oid = i.indexrelid + WHERE ci.relam=(SELECT oid FROM pg_am WHERE amname = ''btree'') + AND ci.relpages > 0 + ) AS idx_data + ) AS ic + JOIN pg_catalog.pg_class ct ON ct.oid = ic.tbloid + LEFT JOIN pg_catalog.pg_attribute a1 ON + ic.indkey[ic.attpos] <> 0 + AND a1.attrelid = ic.tbloid + AND a1.attnum = ic.indkey[ic.attpos] + LEFT JOIN pg_catalog.pg_attribute a2 ON + ic.indkey[ic.attpos] = 0 + AND a2.attrelid = ic.idxoid + AND a2.attnum = ic.attpos + ) i + JOIN pg_catalog.pg_namespace n ON n.oid = i.relnamespace + JOIN pg_catalog.pg_stats s ON s.schemaname = n.nspname + AND s.tablename = i.attrelname + AND s.attname = i.attname + GROUP BY 1,2,3,4,5,6,7,8,9,10,11 + ) AS rows_data_stats + ) AS rows_hdr_pdg_stats +) AS relation_stats +ORDER BY nspname, tblname, idxname; + +CREATE VIEW table_bloat AS /* WARNING: executed with a non-superuser role, the query inspect only tables and materialized view (9.3+) you are granted to read. +* This query is compatible with PostgreSQL 9.0 and more +*/ +SELECT current_database(), schemaname, tblname, bs*tblpages AS real_size, + (tblpages-est_tblpages)*bs AS extra_size, + CASE WHEN tblpages > 0 AND tblpages - est_tblpages > 0 + THEN 100 * (tblpages - est_tblpages)/tblpages::float + ELSE 0 + END AS extra_pct, fillfactor, + CASE WHEN tblpages - est_tblpages_ff > 0 + THEN (tblpages-est_tblpages_ff)*bs + ELSE 0 + END AS bloat_size, + CASE WHEN tblpages > 0 AND tblpages - est_tblpages_ff > 0 + THEN 100 * (tblpages - est_tblpages_ff)/tblpages::float + ELSE 0 + END AS bloat_pct, is_na + -- , tpl_hdr_size, tpl_data_size, (pst).free_percent + (pst).dead_tuple_percent AS real_frag -- (DEBUG INFO) +FROM ( + SELECT ceil( reltuples / ( (bs-page_hdr)/tpl_size ) ) + ceil( toasttuples / 4 ) AS est_tblpages, + ceil( reltuples / ( (bs-page_hdr)*fillfactor/(tpl_size*100) ) ) + ceil( toasttuples / 4 ) AS est_tblpages_ff, + tblpages, fillfactor, bs, tblid, schemaname, tblname, heappages, toastpages, is_na + -- , tpl_hdr_size, tpl_data_size, pgstattuple(tblid) AS pst -- (DEBUG INFO) + FROM ( + SELECT + ( 4 + tpl_hdr_size + tpl_data_size + (2*ma) + - CASE WHEN tpl_hdr_size%ma = 0 THEN ma ELSE tpl_hdr_size%ma END + - CASE WHEN ceil(tpl_data_size)::int%ma = 0 THEN ma ELSE ceil(tpl_data_size)::int%ma END + ) AS tpl_size, bs - page_hdr AS size_per_block, (heappages + toastpages) AS tblpages, heappages, + toastpages, reltuples, toasttuples, bs, page_hdr, tblid, schemaname, tblname, fillfactor, is_na + -- , tpl_hdr_size, tpl_data_size + FROM ( + SELECT + tbl.oid AS tblid, ns.nspname AS schemaname, tbl.relname AS tblname, tbl.reltuples, + tbl.relpages AS heappages, coalesce(toast.relpages, 0) AS toastpages, + coalesce(toast.reltuples, 0) AS toasttuples, + coalesce(substring( + array_to_string(tbl.reloptions, '' '') + FROM ''fillfactor=([0-9]+)'')::smallint, 100) AS fillfactor, + current_setting(''block_size'')::numeric AS bs, + CASE WHEN version()~''mingw32'' OR version()~''64-bit|x86_64|ppc64|ia64|amd64'' THEN 8 ELSE 4 END AS ma, + 24 AS page_hdr, + 23 + CASE WHEN MAX(coalesce(s.null_frac,0)) > 0 THEN ( 7 + count(s.attname) ) / 8 ELSE 0::int END + + CASE WHEN bool_or(att.attname = ''oid'' and att.attnum < 0) THEN 4 ELSE 0 END AS tpl_hdr_size, + sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 0) ) AS tpl_data_size, + bool_or(att.atttypid = ''pg_catalog.name''::regtype) + OR sum(CASE WHEN att.attnum > 0 THEN 1 ELSE 0 END) <> count(s.attname) AS is_na + FROM pg_attribute AS att + JOIN pg_class AS tbl ON att.attrelid = tbl.oid + JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace + LEFT JOIN pg_stats AS s ON s.schemaname=ns.nspname + AND s.tablename = tbl.relname AND s.inherited=false AND s.attname=att.attname + LEFT JOIN pg_class AS toast ON tbl.reltoastrelid = toast.oid + WHERE NOT att.attisdropped + AND tbl.relkind in (''r'',''m'') + GROUP BY 1,2,3,4,5,6,7,8,9,10 + ORDER BY 2,3 + ) AS s + ) AS s2 +) AS s3 +-- WHERE NOT is_na +-- AND tblpages*((pst).free_percent + (pst).dead_tuple_percent)::float4/100 >= 1 +ORDER BY schemaname, tblname; + +-- From https://wiki.postgresql.org/wiki/Lock_dependency_information + +CREATE OR REPLACE VIEW blocking_pid_tree AS +WITH RECURSIVE + lock_composite(requested, current) AS (VALUES + (''AccessShareLock''::text, ''AccessExclusiveLock''::text), + (''RowShareLock''::text, ''ExclusiveLock''::text), + (''RowShareLock''::text, ''AccessExclusiveLock''::text), + (''RowExclusiveLock''::text, ''ShareLock''::text), + (''RowExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''RowExclusiveLock''::text, ''ExclusiveLock''::text), + (''RowExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ShareLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''ShareLock''::text, ''RowExclusiveLock''::text), + (''ShareLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ShareLock''::text, ''ShareRowExclusiveLock''::text), + (''ShareLock''::text, ''ExclusiveLock''::text), + (''ShareLock''::text, ''AccessExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''RowExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''ShareLock''::text), + (''ShareRowExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''ExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''ExclusiveLock''::text, ''RowShareLock''::text), + (''ExclusiveLock''::text, ''RowExclusiveLock''::text), + (''ExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ExclusiveLock''::text, ''ShareLock''::text), + (''ExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''ExclusiveLock''::text, ''ExclusiveLock''::text), + (''ExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''AccessShareLock''::text), + (''AccessExclusiveLock''::text, ''RowShareLock''::text), + (''AccessExclusiveLock''::text, ''RowExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''ShareLock''::text), + (''AccessExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''ExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''AccessExclusiveLock''::text) + ) +, lock AS ( + SELECT pid, + virtualtransaction, + granted, + mode, + (locktype, + CASE locktype + WHEN ''relation'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text) + WHEN ''extend'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text) + WHEN ''page'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text, ''page#''||page::text) + WHEN ''tuple'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text, ''page#''||page::text, ''tuple#''||tuple::text) + WHEN ''transactionid'' THEN transactionid::text + WHEN ''virtualxid'' THEN virtualxid::text + WHEN ''object'' THEN concat_ws('';'', ''class:''||classid::regclass::text, ''objid:''||objid, ''col#''||objsubid) + ELSE concat(''db:''||datname) + END::text) AS target + FROM pg_catalog.pg_locks + LEFT JOIN pg_catalog.pg_database ON (pg_database.oid = pg_locks.database) + ) +, waiting_lock AS ( + SELECT + blocker.pid AS blocker_pid, + blocked.pid AS pid, + concat(blocked.mode,blocked.target) AS lock_target + FROM lock blocker + JOIN lock blocked + ON ( NOT blocked.granted + AND blocker.granted + AND blocked.pid != blocker.pid + AND blocked.target IS NOT DISTINCT FROM blocker.target) + JOIN lock_composite c ON (c.requested = blocked.mode AND c.current = blocker.mode) + ) +, acquired_lock AS ( + WITH waiting AS ( + SELECT lock_target, count(lock_target) AS wait_count FROM waiting_lock GROUP BY lock_target + ) + SELECT + pid, + array_agg(concat(mode,target,'' + ''||wait_count) ORDER BY wait_count DESC NULLS LAST) AS locks_acquired + FROM lock + LEFT JOIN waiting ON waiting.lock_target = concat(mode,target) + WHERE granted + GROUP BY pid + ) +, blocking_lock AS ( + SELECT + ARRAY[date_part(''epoch'', query_start)::int, pid] AS seq, + 0::int AS depth, + -1::int AS blocker_pid, + pid, + concat(''Connect: '',usename,'' '',datname,'' '',coalesce(host(client_addr)||'':''||client_port, ''local'') + , E''\nSQL: '',replace(substr(coalesce(query,''N/A''), 1, 60), E''\n'', '' '') + , E''\nAcquired:\n '' + , array_to_string(locks_acquired[1:5] || + CASE WHEN array_upper(locks_acquired,1) > 5 + THEN ''... ''||(array_upper(locks_acquired,1) - 5)::text||'' more ...'' + END, + E''\n '') + ) AS lock_info, + concat(to_char(query_start, CASE WHEN age(query_start) > ''24h'' THEN ''Day DD Mon'' ELSE ''HH24:MI:SS'' END),E'' started\n'' + ,CASE WHEN wait_event IS NOT NULL THEN ''waiting'' ELSE state END,E''\n'' + ,date_trunc(''second'',age(now(),query_start)),'' ago'' + ) AS lock_state + FROM acquired_lock blocker + LEFT JOIN pg_stat_activity act USING (pid) + WHERE EXISTS + (SELECT ''x'' FROM waiting_lock blocked WHERE blocked.blocker_pid = blocker.pid) + AND NOT EXISTS + (SELECT ''x'' FROM waiting_lock blocked WHERE blocked.pid = blocker.pid) +UNION ALL + SELECT + blocker.seq || blocked.pid, + blocker.depth + 1, + blocker.pid, + blocked.pid, + concat(''Connect: '',usename,'' '',datname,'' '',coalesce(host(client_addr)||'':''||client_port, ''local'') + , E''\nSQL: '',replace(substr(coalesce(query,''N/A''), 1, 60), E''\n'', '' '') + , E''\nWaiting: '',blocked.lock_target + , CASE WHEN locks_acquired IS NOT NULL + THEN E''\nAcquired:\n '' || + array_to_string(locks_acquired[1:5] || + CASE WHEN array_upper(locks_acquired,1) > 5 + THEN ''... ''||(array_upper(locks_acquired,1) - 5)::text||'' more ...'' + END, + E''\n '') + END + ) AS lock_info, + concat(to_char(query_start, CASE WHEN age(query_start) > ''24h'' THEN ''Day DD Mon'' ELSE ''HH24:MI:SS'' END),E'' started\n'' + ,CASE WHEN wait_event IS NOT NULL THEN ''waiting'' ELSE state END,E''\n'' + ,date_trunc(''second'',age(now(),query_start)),'' ago'' + ) AS lock_state + FROM blocking_lock blocker + JOIN waiting_lock blocked + ON (blocked.blocker_pid = blocker.pid) + LEFT JOIN pg_stat_activity act ON (act.pid = blocked.pid) + LEFT JOIN acquired_lock acq ON (acq.pid = blocked.pid) + WHERE blocker.depth < 5 + ) +SELECT concat(lpad(''=> '', 4*depth, '' ''),pid::text) AS "PID" +, lock_info AS "Lock Info" +, lock_state AS "State" +FROM blocking_lock +ORDER BY seq; + + +-- From https://wiki.postgresql.org/wiki/Index_Maintenance + +CREATE VIEW duplicate_indexes AS SELECT pg_size_pretty(sum(pg_relation_size(idx))::bigint) as size, + (array_agg(idx))[1] as idx1, (array_agg(idx))[2] as idx2, + (array_agg(idx))[3] as idx3, (array_agg(idx))[4] as idx4 +FROM ( + SELECT indexrelid::regclass as idx, (indrelid::text ||E''\n''|| indclass::text ||E''\n''|| indkey::text ||E''\n''|| + coalesce(indexprs::text,'''')||E''\n'' || coalesce(indpred::text,'''')) as key + FROM pg_index) sub +GROUP BY key HAVING count(*)>1 +ORDER BY sum(pg_relation_size(idx)) DESC; + +-- From https://wiki.postgresql.org/wiki/Disk_Usage + +CREATE VIEW table_sizes AS WITH RECURSIVE pg_inherit(inhrelid, inhparent) AS + (select inhrelid, inhparent + FROM pg_inherits + UNION + SELECT child.inhrelid, parent.inhparent + FROM pg_inherit child, pg_inherits parent + WHERE child.inhparent = parent.inhrelid), +pg_inherit_short AS (SELECT * FROM pg_inherit WHERE inhparent NOT IN (SELECT inhrelid FROM pg_inherit)) +SELECT table_schema + , TABLE_NAME + , row_estimate + , pg_size_pretty(total_bytes) AS total + , pg_size_pretty(index_bytes) AS INDEX + , pg_size_pretty(toast_bytes) AS toast + , pg_size_pretty(table_bytes) AS TABLE + , total_bytes::float8 / sum(total_bytes) OVER () AS total_size_share + FROM ( + SELECT *, total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes + FROM ( + SELECT c.oid + , nspname AS table_schema + , relname AS TABLE_NAME + , SUM(c.reltuples) OVER (partition BY parent) AS row_estimate + , SUM(pg_total_relation_size(c.oid)) OVER (partition BY parent) AS total_bytes + , SUM(pg_indexes_size(c.oid)) OVER (partition BY parent) AS index_bytes + , SUM(pg_total_relation_size(reltoastrelid)) OVER (partition BY parent) AS toast_bytes + , parent + FROM ( + SELECT pg_class.oid + , reltuples + , relname + , relnamespace + , pg_class.reltoastrelid + , COALESCE(inhparent, pg_class.oid) parent + FROM pg_class + LEFT JOIN pg_inherit_short ON inhrelid = oid + WHERE relkind IN (''r'', ''p'') + ) c + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace + ) a + WHERE oid = parent +) a +ORDER BY total_bytes DESC; + +-- From: https://wiki.postgresql.org/wiki/Index_Maintenance + +CREATE VIEW index_usage AS SELECT + t.schemaname, + t.tablename, + c.reltuples::bigint AS num_rows, + pg_size_pretty(pg_relation_size(c.oid)) AS table_size, + psai.indexrelname AS index_name, + pg_size_pretty(pg_relation_size(i.indexrelid)) AS index_size, + CASE WHEN i.indisunique THEN ''Y'' ELSE ''N'' END AS "unique", + psai.idx_scan AS number_of_scans, + psai.idx_tup_read AS tuples_read, + psai.idx_tup_fetch AS tuples_fetched +FROM + pg_tables t + LEFT JOIN pg_class c ON t.tablename = c.relname + LEFT JOIN pg_index i ON c.oid = i.indrelid + LEFT JOIN pg_stat_all_indexes psai ON i.indexrelid = psai.indexrelid +WHERE + t.schemaname NOT IN (''pg_catalog'', ''information_schema'') +ORDER BY 1, 2; + +-- From: https://blog.devgenius.io/top-useful-sql-queries-for-postgresql-35ff3355d265 + +CREATE VIEW database_sizes AS SELECT pg_database.datname, + pg_size_pretty(pg_database_size(pg_database.datname)) AS size +FROM pg_database +ORDER BY pg_database_size(pg_database.datname) DESC; + +CREATE VIEW schema_sizes AS SELECT A.schemaname, + pg_size_pretty (SUM(pg_relation_size(C.oid))) as table, + pg_size_pretty (SUM(pg_total_relation_size(C.oid)-pg_relation_size(C.oid))) as index, + pg_size_pretty (SUM(pg_total_relation_size(C.oid))) as table_index, + SUM(n_live_tup) +FROM pg_class C +LEFT JOIN pg_namespace N ON (N.oid = C .relnamespace) +INNER JOIN pg_stat_user_tables A ON C.relname = A.relname +WHERE nspname NOT IN (''pg_catalog'', ''information_schema'') +AND C .relkind <> ''i'' +AND nspname !~ ''^pg_toast'' +GROUP BY A.schemaname; + +CREATE VIEW last_vacuum_analyze AS SELECT relname, + last_vacuum, + last_autovacuum + n_mod_since_analyze, + last_analyze, + last_autoanalyze, + analyze_count, + autoanalyze_count + FROM pg_stat_user_tables; + +CREATE VIEW table_row_estimates AS SELECT + schemaname, + relname, + n_live_tup +FROM + pg_stat_user_tables +ORDER BY + n_live_tup DESC; + +-- postgres-meta queries as views from: https://github.com/supabase/postgres-meta + +CREATE VIEW pgmeta_columns AS SELECT + c.oid :: int8 AS table_id, + nc.nspname AS schema, + c.relname AS table, + (c.oid || ''.'' || a.attnum) AS id, + a.attnum AS ordinal_position, + a.attname AS name, + CASE + WHEN a.atthasdef THEN pg_get_expr(ad.adbin, ad.adrelid) + ELSE NULL + END AS default_value, + CASE + WHEN t.typtype = ''d'' THEN CASE + WHEN bt.typelem <> 0 :: oid + AND bt.typlen = -1 THEN ''ARRAY'' + WHEN nbt.nspname = ''pg_catalog'' THEN format_type(t.typbasetype, NULL) + ELSE ''USER-DEFINED'' + END + ELSE CASE + WHEN t.typelem <> 0 :: oid + AND t.typlen = -1 THEN ''ARRAY'' + WHEN nt.nspname = ''pg_catalog'' THEN format_type(a.atttypid, NULL) + ELSE ''USER-DEFINED'' + END + END AS data_type, + COALESCE(bt.typname, t.typname) AS format, + a.attidentity IN (''a'', ''d'') AS is_identity, + CASE + a.attidentity + WHEN ''a'' THEN ''ALWAYS'' + WHEN ''d'' THEN ''BY DEFAULT'' + ELSE NULL + END AS identity_generation, + a.attgenerated IN (''s'') AS is_generated, + NOT ( + a.attnotnull + OR t.typtype = ''d'' AND t.typnotnull + ) AS is_nullable, + ( + c.relkind IN (''r'', ''p'') + OR c.relkind IN (''v'', ''f'') AND pg_column_is_updatable(c.oid, a.attnum, FALSE) + ) AS is_updatable, + uniques.table_id IS NOT NULL AS is_unique, + array_to_json( + array( + SELECT + enumlabel + FROM + pg_catalog.pg_enum enums + WHERE + enums.enumtypid = coalesce(bt.oid, t.oid) + OR enums.enumtypid = coalesce(bt.typelem, t.typelem) + ORDER BY + enums.enumsortorder + ) + ) AS enums, + col_description(c.oid, a.attnum) AS comment +FROM + pg_attribute a + LEFT JOIN pg_attrdef ad ON a.attrelid = ad.adrelid + AND a.attnum = ad.adnum + JOIN ( + pg_class c + JOIN pg_namespace nc ON c.relnamespace = nc.oid + ) ON a.attrelid = c.oid + JOIN ( + pg_type t + JOIN pg_namespace nt ON t.typnamespace = nt.oid + ) ON a.atttypid = t.oid + LEFT JOIN ( + pg_type bt + JOIN pg_namespace nbt ON bt.typnamespace = nbt.oid + ) ON t.typtype = ''d'' + AND t.typbasetype = bt.oid + LEFT JOIN ( + SELECT + conrelid AS table_id, + conkey[1] AS ordinal_position + FROM pg_catalog.pg_constraint + WHERE contype = ''u'' AND cardinality(conkey) = 1 + ) AS uniques ON uniques.table_id = c.oid AND uniques.ordinal_position = a.attnum +WHERE + NOT pg_is_other_temp_schema(nc.oid) + AND a.attnum > 0 + AND NOT a.attisdropped + AND (c.relkind IN (''r'', ''v'', ''m'', ''f'', ''p'')) + AND ( + pg_has_role(c.relowner, ''USAGE'') + OR has_column_privilege( + c.oid, + a.attnum, + ''SELECT, INSERT, UPDATE, REFERENCES'' + ) + ); + +CREATE VIEW pgmeta_config AS SELECT + name, + setting, + category, + TRIM(split_part(category, ''/'', 1)) AS group, + TRIM(split_part(category, ''/'', 2)) AS subgroup, + unit, + short_desc, + extra_desc, + context, + vartype, + source, + min_val, + max_val, + enumvals, + boot_val, + reset_val, + sourcefile, + sourceline, + pending_restart +FROM + pg_settings +ORDER BY + category, + name; + +CREATE VIEW pgmeta_extensions AS SELECT + e.name, + n.nspname AS schema, + e.default_version, + x.extversion AS installed_version, + e.comment +FROM + pg_available_extensions() e(name, default_version, comment) + LEFT JOIN pg_extension x ON e.name = x.extname + LEFT JOIN pg_namespace n ON x.extnamespace = n.oid; + +CREATE VIEW pgmeta_foreign_tables AS SELECT + c.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS name, + obj_description(c.oid) AS comment +FROM + pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE + c.relkind = ''f''; + +CREATE VIEW pgmeta_functions AS with functions as ( + select + *, + -- proargmodes is null when all arg modes are IN + coalesce( + p.proargmodes, + array_fill(''i''::text, array[cardinality(coalesce(p.proallargtypes, p.proargtypes))]) + ) as arg_modes, + -- proargnames is null when all args are unnamed + coalesce( + p.proargnames, + array_fill(''''::text, array[cardinality(coalesce(p.proallargtypes, p.proargtypes))]) + ) as arg_names, + -- proallargtypes is null when all arg modes are IN + coalesce(p.proallargtypes, p.proargtypes) as arg_types, + array_cat( + array_fill(false, array[pronargs - pronargdefaults]), + array_fill(true, array[pronargdefaults])) as arg_has_defaults + from + pg_proc as p + where + p.prokind = ''f'' +) +select + f.oid::int8 as id, + n.nspname as schema, + f.proname as name, + l.lanname as language, + case + when l.lanname = ''internal'' then '''' + else f.prosrc + end as definition, + case + when l.lanname = ''internal'' then f.prosrc + else pg_get_functiondef(f.oid) + end as complete_statement, + coalesce(f_args.args, ''[]'') as args, + pg_get_function_arguments(f.oid) as argument_types, + pg_get_function_identity_arguments(f.oid) as identity_argument_types, + f.prorettype::int8 as return_type_id, + pg_get_function_result(f.oid) as return_type, + nullif(rt.typrelid::int8, 0) as return_type_relation_id, + f.proretset as is_set_returning_function, + case + when f.provolatile = ''i'' then ''IMMUTABLE'' + when f.provolatile = ''s'' then ''STABLE'' + when f.provolatile = ''v'' then ''VOLATILE'' + end as behavior, + f.prosecdef as security_definer, + f_config.config_params as config_params +from + functions f + left join pg_namespace n on f.pronamespace = n.oid + left join pg_language l on f.prolang = l.oid + left join pg_type rt on rt.oid = f.prorettype + left join ( + select + oid, + jsonb_object_agg(param, value) filter (where param is not null) as config_params + from + ( + select + oid, + (string_to_array(unnest(proconfig), ''=''))[1] as param, + (string_to_array(unnest(proconfig), ''=''))[2] as value + from + functions + ) as t + group by + oid + ) f_config on f_config.oid = f.oid + left join ( + select + oid, + jsonb_agg(jsonb_build_object( + ''mode'', t2.mode, + ''name'', name, + ''type_id'', type_id, + ''has_default'', has_default + )) as args + from + ( + select + oid, + unnest(arg_modes) as mode, + unnest(arg_names) as name, + unnest(arg_types)::int8 as type_id, + unnest(arg_has_defaults) as has_default + from + functions + ) as t1, + lateral ( + select + case + when t1.mode = ''i'' then ''in'' + when t1.mode = ''o'' then ''out'' + when t1.mode = ''b'' then ''inout'' + when t1.mode = ''v'' then ''variadic'' + else ''table'' + end as mode + ) as t2 + group by + t1.oid + ) f_args on f_args.oid = f.oid; + +CREATE VIEW pgmeta_materialized_views AS select + c.oid::int8 as id, + n.nspname as schema, + c.relname as name, + c.relispopulated as is_populated, + obj_description(c.oid) as comment +from + pg_class c + join pg_namespace n on n.oid = c.relnamespace +where + c.relkind = ''m''; + +CREATE VIEW pgmeta_policies AS SELECT + pol.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS table, + c.oid :: int8 AS table_id, + pol.polname AS name, + CASE + WHEN pol.polpermissive THEN ''PERMISSIVE'' :: text + ELSE ''RESTRICTIVE'' :: text + END AS action, + CASE + WHEN pol.polroles = ''{0}'' :: oid [] THEN array_to_json( + string_to_array(''public'' :: text, '''' :: text) :: name [] + ) + ELSE array_to_json( + ARRAY( + SELECT + pg_roles.rolname + FROM + pg_roles + WHERE + pg_roles.oid = ANY (pol.polroles) + ORDER BY + pg_roles.rolname + ) + ) + END AS roles, + CASE + pol.polcmd + WHEN ''r'' :: "char" THEN ''SELECT'' :: text + WHEN ''a'' :: "char" THEN ''INSERT'' :: text + WHEN ''w'' :: "char" THEN ''UPDATE'' :: text + WHEN ''d'' :: "char" THEN ''DELETE'' :: text + WHEN ''*'' :: "char" THEN ''ALL'' :: text + ELSE NULL :: text + END AS command, + pg_get_expr(pol.polqual, pol.polrelid) AS definition, + pg_get_expr(pol.polwithcheck, pol.polrelid) AS check +FROM + pg_policy pol + JOIN pg_class c ON c.oid = pol.polrelid + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace; + +CREATE VIEW pgmeta_primary_keys AS SELECT + n.nspname AS schema, + c.relname AS table_name, + a.attname AS name, + c.oid :: int8 AS table_id +FROM + pg_index i, + pg_class c, + pg_attribute a, + pg_namespace n +WHERE + i.indrelid = c.oid + AND c.relnamespace = n.oid + AND a.attrelid = c.oid + AND a.attnum = ANY (i.indkey) + AND i.indisprimary; + +CREATE VIEW pgmeta_publications AS SELECT + p.oid :: int8 AS id, + p.pubname AS name, + p.pubowner::regrole::text AS owner, + p.pubinsert AS publish_insert, + p.pubupdate AS publish_update, + p.pubdelete AS publish_delete, + p.pubtruncate AS publish_truncate, + CASE + WHEN p.puballtables THEN NULL + ELSE pr.tables + END AS tables +FROM + pg_catalog.pg_publication AS p + LEFT JOIN LATERAL ( + SELECT + COALESCE( + array_agg( + json_build_object( + ''id'', + c.oid :: int8, + ''name'', + c.relname, + ''schema'', + nc.nspname + ) + ), + ''{}'' + ) AS tables + FROM + pg_catalog.pg_publication_rel AS pr + JOIN pg_class AS c ON pr.prrelid = c.oid + join pg_namespace as nc on c.relnamespace = nc.oid + WHERE + pr.prpubid = p.oid + ) AS pr ON 1 = 1; + +CREATE VIEW pgmeta_relationships AS SELECT + c.oid :: int8 AS id, + c.conname AS constraint_name, + nsa.nspname AS source_schema, + csa.relname AS source_table_name, + sa.attname AS source_column_name, + nta.nspname AS target_table_schema, + cta.relname AS target_table_name, + ta.attname AS target_column_name +FROM + pg_constraint c + JOIN ( + pg_attribute sa + JOIN pg_class csa ON sa.attrelid = csa.oid + JOIN pg_namespace nsa ON csa.relnamespace = nsa.oid + ) ON sa.attrelid = c.conrelid + AND sa.attnum = ANY (c.conkey) + JOIN ( + pg_attribute ta + JOIN pg_class cta ON ta.attrelid = cta.oid + JOIN pg_namespace nta ON cta.relnamespace = nta.oid + ) ON ta.attrelid = c.confrelid + AND ta.attnum = ANY (c.confkey) +WHERE + c.contype = ''f''; + +CREATE VIEW pgmeta_roles AS -- TODO: Consider using pg_authid vs. pg_roles for unencrypted password field +SELECT + oid :: int8 AS id, + rolname AS name, + rolsuper AS is_superuser, + rolcreatedb AS can_create_db, + rolcreaterole AS can_create_role, + rolinherit AS inherit_role, + rolcanlogin AS can_login, + rolreplication AS is_replication_role, + rolbypassrls AS can_bypass_rls, + ( + SELECT + COUNT(*) + FROM + pg_stat_activity + WHERE + pg_roles.rolname = pg_stat_activity.usename + ) AS active_connections, + CASE WHEN rolconnlimit = -1 THEN current_setting(''max_connections'') :: int8 + ELSE rolconnlimit + END AS connection_limit, + rolpassword AS password, + rolvaliduntil AS valid_until, + rolconfig AS config +FROM + pg_roles; + +CREATE VIEW pgmeta_schemas AS select + n.oid::int8 as id, + n.nspname as name, + u.rolname as owner +from + pg_namespace n, + pg_roles u +where + n.nspowner = u.oid + and ( + pg_has_role(n.nspowner, ''USAGE'') + or has_schema_privilege(n.oid, ''CREATE, USAGE'') + ) + and not pg_catalog.starts_with(n.nspname, ''pg_temp_'') + and not pg_catalog.starts_with(n.nspname, ''pg_toast_temp_''); + +CREATE VIEW pgmeta_tables AS SELECT + c.oid :: int8 AS id, + nc.nspname AS schema, + c.relname AS name, + c.relrowsecurity AS rls_enabled, + c.relforcerowsecurity AS rls_forced, + CASE + WHEN c.relreplident = ''d'' THEN ''DEFAULT'' + WHEN c.relreplident = ''i'' THEN ''INDEX'' + WHEN c.relreplident = ''f'' THEN ''FULL'' + ELSE ''NOTHING'' + END AS replica_identity, + pg_total_relation_size(format(''%I.%I'', nc.nspname, c.relname)) :: int8 AS bytes, + pg_size_pretty( + pg_total_relation_size(format(''%I.%I'', nc.nspname, c.relname)) + ) AS size, + pg_stat_get_live_tuples(c.oid) AS live_rows_estimate, + pg_stat_get_dead_tuples(c.oid) AS dead_rows_estimate, + obj_description(c.oid) AS comment +FROM + pg_namespace nc + JOIN pg_class c ON nc.oid = c.relnamespace +WHERE + c.relkind IN (''r'', ''p'') + AND NOT pg_is_other_temp_schema(nc.oid) + AND ( + pg_has_role(c.relowner, ''USAGE'') + OR has_table_privilege( + c.oid, + ''SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER'' + ) + OR has_any_column_privilege(c.oid, ''SELECT, INSERT, UPDATE, REFERENCES'') + ); + + +CREATE VIEW pgmeta_triggers AS SELECT + pg_t.oid AS id, + pg_t.tgrelid AS table_id, + CASE + WHEN pg_t.tgenabled = ''D'' THEN ''DISABLED'' + WHEN pg_t.tgenabled = ''O'' THEN ''ORIGIN'' + WHEN pg_t.tgenabled = ''R'' THEN ''REPLICA'' + WHEN pg_t.tgenabled = ''A'' THEN ''ALWAYS'' + END AS enabled_mode, + ( + STRING_TO_ARRAY( + ENCODE(pg_t.tgargs, ''escape''), ''\000'' + ) + )[:pg_t.tgnargs] AS function_args, + is_t.trigger_name AS name, + is_t.event_object_table AS table, + is_t.event_object_schema AS schema, + is_t.action_condition AS condition, + is_t.action_orientation AS orientation, + is_t.action_timing AS activation, + ARRAY_AGG(is_t.event_manipulation)::text[] AS events, + pg_p.proname AS function_name, + pg_n.nspname AS function_schema +FROM + pg_trigger AS pg_t +JOIN + pg_class AS pg_c +ON pg_t.tgrelid = pg_c.oid +JOIN information_schema.triggers AS is_t +ON is_t.trigger_name = pg_t.tgname +AND pg_c.relname = is_t.event_object_table +JOIN pg_proc AS pg_p +ON pg_t.tgfoid = pg_p.oid +JOIN pg_namespace AS pg_n +ON pg_p.pronamespace = pg_n.oid +GROUP BY + pg_t.oid, + pg_t.tgrelid, + pg_t.tgenabled, + pg_t.tgargs, + pg_t.tgnargs, + is_t.trigger_name, + is_t.event_object_table, + is_t.event_object_schema, + is_t.action_condition, + is_t.action_orientation, + is_t.action_timing, + pg_p.proname, + pg_n.nspname; + + +CREATE VIEW pgmeta_types AS select + t.oid::int8 as id, + t.typname as name, + n.nspname as schema, + format_type (t.oid, null) as format, + coalesce(t_enums.enums, ''[]'') as enums, + coalesce(t_attributes.attributes, ''[]'') as attributes, + obj_description (t.oid, ''pg_type'') as comment +from + pg_type t + left join pg_namespace n on n.oid = t.typnamespace + left join ( + select + enumtypid, + jsonb_agg(enumlabel order by enumsortorder) as enums + from + pg_enum + group by + enumtypid + ) as t_enums on t_enums.enumtypid = t.oid + left join ( + select + oid, + jsonb_agg( + jsonb_build_object(''name'', a.attname, ''type_id'', a.atttypid::int8) + order by a.attnum asc + ) as attributes + from + pg_class c + join pg_attribute a on a.attrelid = c.oid + where + c.relkind = ''c'' and not a.attisdropped + group by + c.oid + ) as t_attributes on t_attributes.oid = t.typrelid +where + ( + t.typrelid = 0 + or ( + select + c.relkind = ''c'' + from + pg_class c + where + c.oid = t.typrelid + ) + ); + +CREATE VIEW pgmeta_version AS SELECT + version(), + current_setting(''server_version_num'') :: int8 AS version_number, + ( + SELECT + COUNT(*) AS active_connections + FROM + pg_stat_activity + ) AS active_connections, + current_setting(''max_connections'') :: int8 AS max_connections; + +CREATE VIEW pgmeta_views AS SELECT + c.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS name, + -- See definition of information_schema.views + (pg_relation_is_updatable(c.oid, false) & 20) = 20 AS is_updatable, + obj_description(c.oid) AS comment +FROM + pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE + c.relkind = ''v''; +', + ' +# michelp@adminpack + +A Trusted Language Extension containing a variety of useful databse admin queries. + +Postgres database admins are often faced with a variety of issues that +require them introspecting the database state. This extension +contains a mixed-bag of views that reflect useful information for +admins. + +## Installation + +Using dbdev: + +``` +postgres=> select dbdev.install(''michelp@adminpack''); + install +--------- + t +(1 row) + +postgres=> create schema adminpack; +CREATE SCHEMA +postgres=> create extension "michelp@adminpack" with schema adminpack; +CREATE EXTENSION +``` + +This will the extension into the `adminpack` schema, substitute with +some other schema if you want it to go somewhere else. + +## Index Bloat + +Index updates and querying can end up getting slower and slower over +time due to what''s called "index bloat". This is where frequent +updates and deletes can cause space in index data structures to go +unused. Understanding when this is happening is not obvious, so there +is a rather complex query you can run to detect it. + +`index_bloat` + +| Column | Type | +|------------------|------------------| +| current_database | name | +| schemaname | name | +| tblname | name | +| idxname | name | +| real_size | numeric | +| extra_size | numeric | +| extra_pct | double precision | +| fillfactor | integer | +| bloat_size | double precision | +| bloat_pct | double precision | +| is_na | boolean | + + +## Table Bloat + +Like index bloat, tables with high update and delete rates can also +end up containing lots of allocated but unused space. This query +shows which tables have the most bloat. + +`table_bloat` + +| Column | Type | +|------------------|------------------| +| current_database | name | +| schemaname | name | +| tblname | name | +| real_size | numeric | +| extra_size | double precision | +| extra_pct | double precision | +| fillfactor | integer | +| bloat_size | double precision | +| bloat_pct | double precision | +| is_na | boolean | + + +## Blocking PID Tree + +Postgres queries can block each other, and this blocking relationship +can form a tree, where A blocks B, which blocks C, and so on. This +query formats blocking queries into a tree structure so it''s easy to +see what query is causing the blockage. + +`blocking_pid_tree` + +| Column | Type | +|-----------|------| +| PID | text | +| Lock Info | text | +| State | text | + + +## Duplicate Indexes + +If a table contains duplicate indexes, then unecessary work is done +updating and storing them, this query will show up to 4 duplicate +indexes per table if they exist. + +`duplicate_indexes` + +| Column | Type | +|--------|----------| +| size | text | +| idx1 | regclass | +| idx2 | regclass | +| idx3 | regclass | +| idx4 | regclass | + + +## Table Sizes + +This view shows tables and their sizes. + +`table_sizes` + +| Column | Type | +|------------------|------------------| +| table_schema | name | +| table_name | name | +| row_estimate | real | +| total | text | +| index | text | +| toast | text | +| table | text | +| total_size_share | double precision | + + +## Schema Sizes + +This view shows schemas and their sizes, which is the sum of the sizes +of all the tables and indexes in the schema. + +`schema_sizes` + +| Column | Type | +|-------------|---------| +| schemaname | name | +| table | text | +| index | text | +| table_index | text | +| sum | numeric | + + +## Index Usage + +This view shows index size and usage. An unused index still needs to +be updated and that takes time an storage, so it''s a good candidate to +drop. + +`index_usage` + +| Column | Type | +|-----------------|--------| +| schemaname | name | +| tablename | name | +| num_rows | bigint | +| table_size | text | +| index_name | name | +| index_size | text | +| unique | text | +| number_of_scans | bigint | +| tuples_read | bigint | +| tuples_fetched | bigint | + + +## Last Vacuum Analyze + +This views shows the last time a table was vacuumed an analyzed. + +`last_vacuum_analyze` + +| Column | Type | +|---------------------|--------------------------| +| relname | name | +| last_vacuum | timestamp with time zone | +| n_mod_since_analyze | timestamp with time zone | +| last_analyze | timestamp with time zone | +| last_autoanalyze | timestamp with time zone | +| analyze_count | bigint | +| autoanalyze_count | bigint | + +## Table Row Estimates + +This view shows estimates for the number of rows in a table. This is +just an estimate and depends on up to date table statistics. + +`table_row_estimates` + +| Column | Type | +|------------|--------| +| schemaname | name | +| relname | name | +| n_live_tup | bigint | + +## PGMeta Columns + +This view shows infromation for the columns of tables in the system. + +`pgmeta_columns` + +| Column | Type | +|---------------------|----------| +| table_id | bigint | +| schema | name | +| table | name | +| id | text | +| ordinal_position | smallint | +| name | name | +| default_value | text | +| data_type | text | +| format | name | +| is_identity | boolean | +| identity_generation | text | +| is_generated | boolean | +| is_nullable | boolean | +| is_updatable | boolean | +| is_unique | boolean | +| enums | json | +| comment | text | + +## PGMeta Config + +This views shows the configuration of the database. + +`pgmeta_config` + +| Column | Type | +|-----------------|---------| +| name | text | +| setting | text | +| category | text | +| group | text | +| subgroup | text | +| unit | text | +| short_desc | text | +| extra_desc | text | +| context | text | +| vartype | text | +| source | text | +| min_val | text | +| max_val | text | +| enumvals | text[] | +| boot_val | text | +| reset_val | text | +| sourcefile | text | +| sourceline | integer | +| pending_restart | boolean | + +## PGMeta Extensions + +This view shows installed extensions in the database. + +`pgmeta_extensions` + +| Column | Type | +|-------------------|------| +| name | name | +| schema | name | +| default_version | text | +| installed_version | text | +| comment | text | + +## PGMeta Foreign Tables + +This view shows foreign tables in the database. + +`pgmeta_foreign_tables` + +| Column | Type | +|---------|--------| +| id | bigint | +| schema | name | +| name | name | +| comment | text | + +## PGMeta Functions + +This view shows functions in the database. + +`pgmeta_functions` + +| Column | Type | +|---------------------------|---------| +| id | bigint | +| schema | name | +| name | name | +| language | name | +| definition | text | +| complete_statement | text | +| args | jsonb | +| argument_types | text | +| identity_argument_types | text | +| return_type_id | bigint | +| return_type | text | +| return_type_relation_id | bigint | +| is_set_returning_function | boolean | +| behavior | text | +| security_definer | boolean | +| config_params | jsonb | + +## PGMeta Materialized Views + +This view shows materialized views in the database. + +`pgmeta_materialized_views` + +| Column | Type | +|--------------|---------| +| id | bigint | +| schema | name | +| name | name | +| is_populated | boolean | +| comment | text | + +## PGMeta Policies + +This view shows Row Level Security Policies in the database. + +`pgmeta_policies` + +| Column | Type | +|------------|--------| +| id | bigint | +| schema | name | +| table | name | +| table_id | bigint | +| name | name | +| action | text | +| roles | json | +| command | text | +| definition | text | +| check | text | + +## PGMeta Primary Keys + +This view shows primary keys in the database. + +`pgmeta_primary_keys` + +| Column | Type | +|------------|--------| +| schema | name | +| table_name | name | +| name | name | +| table_id | bigint | + +## PGMeta Publications + +This view shows logical replication publishers in the database. + +`pgmeta_publications` + +| Column | Type | +|------------------|---------| +| id | bigint | +| name | name | +| owner | text | +| publish_insert | boolean | +| publish_update | boolean | +| publish_delete | boolean | +| publish_truncate | boolean | +| tables | json[] | + +## PGMeta Relationships + +This view shows foreign key relationships in the database. + +`pgmeta_relationships` + +| Column | Type | +|---------------------|--------| +| id | bigint | +| constraint_name | name | +| source_schema | name | +| source_table_name | name | +| source_column_name | name | +| target_table_schema | name | +| target_table_name | name | +| target_column_name | name | + +## PGMeta Roles + +This view shows roles in the database system. Note that roles are +global objects and apply to all databases. + +`pgmeta_roles` + +| Column | Type | +|---------------------|--------------------------| +| id | bigint | +| name | name | +| is_superuser | boolean | +| can_create_db | boolean | +| can_create_role | boolean | +| inherit_role | boolean | +| can_login | boolean | +| is_replication_role | boolean | +| can_bypass_rls | boolean | +| active_connections | bigint | +| connection_limit | bigint | +| password | text | +| valid_until | timestamp with time zone | +| config | text[] | + +## PGMeta Schemas + +This view shows all schemas in the database. + +`pgmeta_schemas` + +| Column | Type | +|--------|--------| +| id | bigint | +| name | name | +| owner | name | + +## PGMeta Tables + +This view shows all tables in the database. + +`pgmeta_tables` + +| Column | Type | +|--------------------|---------| +| id | bigint | +| schema | name | +| name | name | +| rls_enabled | boolean | +| rls_forced | boolean | +| replica_identity | text | +| bytes | bigint | +| size | text | +| live_rows_estimate | bigint | +| dead_rows_estimate | bigint | +| comment | text | + +## PGMeta Triggers + +This view shows all triggers in the database. + +`pgmeta_triggers` + +| Column | Type | +|-----------------|-----------------------------------| +| id | oid | +| table_id | oid | +| enabled_mode | text | +| function_args | text[] | +| name | information_schema.sql_identifier | +| table | information_schema.sql_identifier | +| schema | information_schema.sql_identifier | +| condition | information_schema.character_data | +| orientation | information_schema.character_data | +| activation | information_schema.character_data | +| events | text[] | +| function_name | name | +| function_schema | name | + +## PGMeta Types + +This view shows all types in the database. + +`pgmeta_types` + +| Column | Type | +|------------|--------| +| id | bigint | +| name | name | +| schema | name | +| format | text | +| enums | jsonb | +| attributes | jsonb | +| comment | text | + +## PGMeta Version + +This view shows the current database version. + +`pgmeta_version` + +| Column | Type | +|--------------------|--------| +| version | text | +| version_number | bigint | +| active_connections | bigint | +| max_connections | bigint | + +## PGMeta Views + +This view shows all views in the database. + +`pgmeta_views` + +| Column | Type | +|--------------|---------| +| id | bigint | +| schema | name | +| name | name | +| is_updatable | boolean | +| comment | text | +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112942_michelp@adminpack-0.0.2_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112942_michelp@adminpack-0.0.2_80.snap new file mode 100644 index 000000000..510275528 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207112942_michelp@adminpack-0.0.2_80.snap @@ -0,0 +1,1571 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231207112942_michelp@adminpack-0.0.2.sql +--- +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_alias = 'michelp@adminpack' + ), + (0, 0, 2), + ' +-- From: https://github.com/ioguix/pgsql-bloat-estimation + +-- Copyright (c) 2015-2019, Jehan-Guillaume (ioguix) de Rorthais +-- All rights reserved. + +-- Redistribution and use in source and binary forms, with or without +-- modification, are permitted provided that the following conditions are met: + +-- * Redistributions of source code must retain the above copyright notice, this +-- list of conditions and the following disclaimer. + +-- * Redistributions in binary form must reproduce the above copyright notice, +-- this list of conditions and the following disclaimer in the documentation +-- and/or other materials provided with the distribution. + +-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +-- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +-- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +-- DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +-- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +-- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +-- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +-- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +-- OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +-- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-- WARNING: executed with a non-superuser role, the query inspect only index on tables you are granted to read. +-- WARNING: rows with is_na = ''t'' are known to have bad statistics ("name" type is not supported). +-- This query is compatible with PostgreSQL 8.2 and after +CREATE VIEW index_bloat AS SELECT current_database(), nspname AS schemaname, tblname, idxname, bs*(relpages)::bigint AS real_size, + bs*(relpages-est_pages)::bigint AS extra_size, + 100 * (relpages-est_pages)::float / relpages AS extra_pct, + fillfactor, + CASE WHEN relpages > est_pages_ff + THEN bs*(relpages-est_pages_ff) + ELSE 0 + END AS bloat_size, + 100 * (relpages-est_pages_ff)::float / relpages AS bloat_pct, + is_na + -- , 100-(pst).avg_leaf_density AS pst_avg_bloat, est_pages, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples, relpages -- (DEBUG INFO) +FROM ( + SELECT coalesce(1 + + ceil(reltuples/floor((bs-pageopqdata-pagehdr)/(4+nulldatahdrwidth)::float)), 0 -- ItemIdData size + computed avg size of a tuple (nulldatahdrwidth) + ) AS est_pages, + coalesce(1 + + ceil(reltuples/floor((bs-pageopqdata-pagehdr)*fillfactor/(100*(4+nulldatahdrwidth)::float))), 0 + ) AS est_pages_ff, + bs, nspname, tblname, idxname, relpages, fillfactor, is_na + -- , pgstatindex(idxoid) AS pst, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples -- (DEBUG INFO) + FROM ( + SELECT maxalign, bs, nspname, tblname, idxname, reltuples, relpages, idxoid, fillfactor, + ( index_tuple_hdr_bm + + maxalign - CASE -- Add padding to the index tuple header to align on MAXALIGN + WHEN index_tuple_hdr_bm%maxalign = 0 THEN maxalign + ELSE index_tuple_hdr_bm%maxalign + END + + nulldatawidth + maxalign - CASE -- Add padding to the data to align on MAXALIGN + WHEN nulldatawidth = 0 THEN 0 + WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign + ELSE nulldatawidth::integer%maxalign + END + )::numeric AS nulldatahdrwidth, pagehdr, pageopqdata, is_na + -- , index_tuple_hdr_bm, nulldatawidth -- (DEBUG INFO) + FROM ( + SELECT n.nspname, i.tblname, i.idxname, i.reltuples, i.relpages, + i.idxoid, i.fillfactor, current_setting(''block_size'')::numeric AS bs, + CASE -- MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?) + WHEN version() ~ ''mingw32'' OR version() ~ ''64-bit|x86_64|ppc64|ia64|amd64'' THEN 8 + ELSE 4 + END AS maxalign, + /* per page header, fixed size: 20 for 7.X, 24 for others */ + 24 AS pagehdr, + /* per page btree opaque data */ + 16 AS pageopqdata, + /* per tuple header: add IndexAttributeBitMapData if some cols are null-able */ + CASE WHEN max(coalesce(s.null_frac,0)) = 0 + THEN 8 -- IndexTupleData size + ELSE 8 + (( 32 + 8 - 1 ) / 8) -- IndexTupleData size + IndexAttributeBitMapData size ( max num filed per index + 8 - 1 /8) + END AS index_tuple_hdr_bm, + /* data len: we remove null values save space using it fractionnal part from stats */ + sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) AS nulldatawidth, + max( CASE WHEN i.atttypid = ''pg_catalog.name''::regtype THEN 1 ELSE 0 END ) > 0 AS is_na + FROM ( + SELECT ct.relname AS tblname, ct.relnamespace, ic.idxname, ic.attpos, ic.indkey, ic.indkey[ic.attpos], ic.reltuples, ic.relpages, ic.tbloid, ic.idxoid, ic.fillfactor, + coalesce(a1.attnum, a2.attnum) AS attnum, coalesce(a1.attname, a2.attname) AS attname, coalesce(a1.atttypid, a2.atttypid) AS atttypid, + CASE WHEN a1.attnum IS NULL + THEN ic.idxname + ELSE ct.relname + END AS attrelname + FROM ( + SELECT idxname, reltuples, relpages, tbloid, idxoid, fillfactor, indkey, + pg_catalog.generate_series(1,indnatts) AS attpos + FROM ( + SELECT ci.relname AS idxname, ci.reltuples, ci.relpages, i.indrelid AS tbloid, + i.indexrelid AS idxoid, + coalesce(substring( + array_to_string(ci.reloptions, '' '') + from ''fillfactor=([0-9]+)'')::smallint, 90) AS fillfactor, + i.indnatts, + pg_catalog.string_to_array(pg_catalog.textin( + pg_catalog.int2vectorout(i.indkey)),'' '')::int[] AS indkey + FROM pg_catalog.pg_index i + JOIN pg_catalog.pg_class ci ON ci.oid = i.indexrelid + WHERE ci.relam=(SELECT oid FROM pg_am WHERE amname = ''btree'') + AND ci.relpages > 0 + ) AS idx_data + ) AS ic + JOIN pg_catalog.pg_class ct ON ct.oid = ic.tbloid + LEFT JOIN pg_catalog.pg_attribute a1 ON + ic.indkey[ic.attpos] <> 0 + AND a1.attrelid = ic.tbloid + AND a1.attnum = ic.indkey[ic.attpos] + LEFT JOIN pg_catalog.pg_attribute a2 ON + ic.indkey[ic.attpos] = 0 + AND a2.attrelid = ic.idxoid + AND a2.attnum = ic.attpos + ) i + JOIN pg_catalog.pg_namespace n ON n.oid = i.relnamespace + JOIN pg_catalog.pg_stats s ON s.schemaname = n.nspname + AND s.tablename = i.attrelname + AND s.attname = i.attname + GROUP BY 1,2,3,4,5,6,7,8,9,10,11 + ) AS rows_data_stats + ) AS rows_hdr_pdg_stats +) AS relation_stats +ORDER BY nspname, tblname, idxname; + +CREATE VIEW table_bloat AS /* WARNING: executed with a non-superuser role, the query inspect only tables and materialized view (9.3+) you are granted to read. +* This query is compatible with PostgreSQL 9.0 and more +*/ +SELECT current_database(), schemaname, tblname, bs*tblpages AS real_size, + (tblpages-est_tblpages)*bs AS extra_size, + CASE WHEN tblpages > 0 AND tblpages - est_tblpages > 0 + THEN 100 * (tblpages - est_tblpages)/tblpages::float + ELSE 0 + END AS extra_pct, fillfactor, + CASE WHEN tblpages - est_tblpages_ff > 0 + THEN (tblpages-est_tblpages_ff)*bs + ELSE 0 + END AS bloat_size, + CASE WHEN tblpages > 0 AND tblpages - est_tblpages_ff > 0 + THEN 100 * (tblpages - est_tblpages_ff)/tblpages::float + ELSE 0 + END AS bloat_pct, is_na + -- , tpl_hdr_size, tpl_data_size, (pst).free_percent + (pst).dead_tuple_percent AS real_frag -- (DEBUG INFO) +FROM ( + SELECT ceil( reltuples / ( (bs-page_hdr)/tpl_size ) ) + ceil( toasttuples / 4 ) AS est_tblpages, + ceil( reltuples / ( (bs-page_hdr)*fillfactor/(tpl_size*100) ) ) + ceil( toasttuples / 4 ) AS est_tblpages_ff, + tblpages, fillfactor, bs, tblid, schemaname, tblname, heappages, toastpages, is_na + -- , tpl_hdr_size, tpl_data_size, pgstattuple(tblid) AS pst -- (DEBUG INFO) + FROM ( + SELECT + ( 4 + tpl_hdr_size + tpl_data_size + (2*ma) + - CASE WHEN tpl_hdr_size%ma = 0 THEN ma ELSE tpl_hdr_size%ma END + - CASE WHEN ceil(tpl_data_size)::int%ma = 0 THEN ma ELSE ceil(tpl_data_size)::int%ma END + ) AS tpl_size, bs - page_hdr AS size_per_block, (heappages + toastpages) AS tblpages, heappages, + toastpages, reltuples, toasttuples, bs, page_hdr, tblid, schemaname, tblname, fillfactor, is_na + -- , tpl_hdr_size, tpl_data_size + FROM ( + SELECT + tbl.oid AS tblid, ns.nspname AS schemaname, tbl.relname AS tblname, tbl.reltuples, + tbl.relpages AS heappages, coalesce(toast.relpages, 0) AS toastpages, + coalesce(toast.reltuples, 0) AS toasttuples, + coalesce(substring( + array_to_string(tbl.reloptions, '' '') + FROM ''fillfactor=([0-9]+)'')::smallint, 100) AS fillfactor, + current_setting(''block_size'')::numeric AS bs, + CASE WHEN version()~''mingw32'' OR version()~''64-bit|x86_64|ppc64|ia64|amd64'' THEN 8 ELSE 4 END AS ma, + 24 AS page_hdr, + 23 + CASE WHEN MAX(coalesce(s.null_frac,0)) > 0 THEN ( 7 + count(s.attname) ) / 8 ELSE 0::int END + + CASE WHEN bool_or(att.attname = ''oid'' and att.attnum < 0) THEN 4 ELSE 0 END AS tpl_hdr_size, + sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 0) ) AS tpl_data_size, + bool_or(att.atttypid = ''pg_catalog.name''::regtype) + OR sum(CASE WHEN att.attnum > 0 THEN 1 ELSE 0 END) <> count(s.attname) AS is_na + FROM pg_attribute AS att + JOIN pg_class AS tbl ON att.attrelid = tbl.oid + JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace + LEFT JOIN pg_stats AS s ON s.schemaname=ns.nspname + AND s.tablename = tbl.relname AND s.inherited=false AND s.attname=att.attname + LEFT JOIN pg_class AS toast ON tbl.reltoastrelid = toast.oid + WHERE NOT att.attisdropped + AND tbl.relkind in (''r'',''m'') + GROUP BY 1,2,3,4,5,6,7,8,9,10 + ORDER BY 2,3 + ) AS s + ) AS s2 +) AS s3 +-- WHERE NOT is_na +-- AND tblpages*((pst).free_percent + (pst).dead_tuple_percent)::float4/100 >= 1 +ORDER BY schemaname, tblname; + +-- From https://wiki.postgresql.org/wiki/Lock_dependency_information + +CREATE OR REPLACE VIEW blocking_pid_tree AS +WITH RECURSIVE + lock_composite(requested, current) AS (VALUES + (''AccessShareLock''::text, ''AccessExclusiveLock''::text), + (''RowShareLock''::text, ''ExclusiveLock''::text), + (''RowShareLock''::text, ''AccessExclusiveLock''::text), + (''RowExclusiveLock''::text, ''ShareLock''::text), + (''RowExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''RowExclusiveLock''::text, ''ExclusiveLock''::text), + (''RowExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ShareLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''ExclusiveLock''::text), + (''ShareUpdateExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''ShareLock''::text, ''RowExclusiveLock''::text), + (''ShareLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ShareLock''::text, ''ShareRowExclusiveLock''::text), + (''ShareLock''::text, ''ExclusiveLock''::text), + (''ShareLock''::text, ''AccessExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''RowExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''ShareLock''::text), + (''ShareRowExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''ExclusiveLock''::text), + (''ShareRowExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''ExclusiveLock''::text, ''RowShareLock''::text), + (''ExclusiveLock''::text, ''RowExclusiveLock''::text), + (''ExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''ExclusiveLock''::text, ''ShareLock''::text), + (''ExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''ExclusiveLock''::text, ''ExclusiveLock''::text), + (''ExclusiveLock''::text, ''AccessExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''AccessShareLock''::text), + (''AccessExclusiveLock''::text, ''RowShareLock''::text), + (''AccessExclusiveLock''::text, ''RowExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''ShareUpdateExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''ShareLock''::text), + (''AccessExclusiveLock''::text, ''ShareRowExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''ExclusiveLock''::text), + (''AccessExclusiveLock''::text, ''AccessExclusiveLock''::text) + ) +, lock AS ( + SELECT pid, + virtualtransaction, + granted, + mode, + (locktype, + CASE locktype + WHEN ''relation'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text) + WHEN ''extend'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text) + WHEN ''page'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text, ''page#''||page::text) + WHEN ''tuple'' THEN concat_ws('';'', ''db:''||datname, ''rel:''||relation::regclass::text, ''page#''||page::text, ''tuple#''||tuple::text) + WHEN ''transactionid'' THEN transactionid::text + WHEN ''virtualxid'' THEN virtualxid::text + WHEN ''object'' THEN concat_ws('';'', ''class:''||classid::regclass::text, ''objid:''||objid, ''col#''||objsubid) + ELSE concat(''db:''||datname) + END::text) AS target + FROM pg_catalog.pg_locks + LEFT JOIN pg_catalog.pg_database ON (pg_database.oid = pg_locks.database) + ) +, waiting_lock AS ( + SELECT + blocker.pid AS blocker_pid, + blocked.pid AS pid, + concat(blocked.mode,blocked.target) AS lock_target + FROM lock blocker + JOIN lock blocked + ON ( NOT blocked.granted + AND blocker.granted + AND blocked.pid != blocker.pid + AND blocked.target IS NOT DISTINCT FROM blocker.target) + JOIN lock_composite c ON (c.requested = blocked.mode AND c.current = blocker.mode) + ) +, acquired_lock AS ( + WITH waiting AS ( + SELECT lock_target, count(lock_target) AS wait_count FROM waiting_lock GROUP BY lock_target + ) + SELECT + pid, + array_agg(concat(mode,target,'' + ''||wait_count) ORDER BY wait_count DESC NULLS LAST) AS locks_acquired + FROM lock + LEFT JOIN waiting ON waiting.lock_target = concat(mode,target) + WHERE granted + GROUP BY pid + ) +, blocking_lock AS ( + SELECT + ARRAY[date_part(''epoch'', query_start)::int, pid] AS seq, + 0::int AS depth, + -1::int AS blocker_pid, + pid, + concat(''Connect: '',usename,'' '',datname,'' '',coalesce(host(client_addr)||'':''||client_port, ''local'') + , E''\nSQL: '',replace(substr(coalesce(query,''N/A''), 1, 60), E''\n'', '' '') + , E''\nAcquired:\n '' + , array_to_string(locks_acquired[1:5] || + CASE WHEN array_upper(locks_acquired,1) > 5 + THEN ''... ''||(array_upper(locks_acquired,1) - 5)::text||'' more ...'' + END, + E''\n '') + ) AS lock_info, + concat(to_char(query_start, CASE WHEN age(query_start) > ''24h'' THEN ''Day DD Mon'' ELSE ''HH24:MI:SS'' END),E'' started\n'' + ,CASE WHEN wait_event IS NOT NULL THEN ''waiting'' ELSE state END,E''\n'' + ,date_trunc(''second'',age(now(),query_start)),'' ago'' + ) AS lock_state + FROM acquired_lock blocker + LEFT JOIN pg_stat_activity act USING (pid) + WHERE EXISTS + (SELECT ''x'' FROM waiting_lock blocked WHERE blocked.blocker_pid = blocker.pid) + AND NOT EXISTS + (SELECT ''x'' FROM waiting_lock blocked WHERE blocked.pid = blocker.pid) +UNION ALL + SELECT + blocker.seq || blocked.pid, + blocker.depth + 1, + blocker.pid, + blocked.pid, + concat(''Connect: '',usename,'' '',datname,'' '',coalesce(host(client_addr)||'':''||client_port, ''local'') + , E''\nSQL: '',replace(substr(coalesce(query,''N/A''), 1, 60), E''\n'', '' '') + , E''\nWaiting: '',blocked.lock_target + , CASE WHEN locks_acquired IS NOT NULL + THEN E''\nAcquired:\n '' || + array_to_string(locks_acquired[1:5] || + CASE WHEN array_upper(locks_acquired,1) > 5 + THEN ''... ''||(array_upper(locks_acquired,1) - 5)::text||'' more ...'' + END, + E''\n '') + END + ) AS lock_info, + concat(to_char(query_start, CASE WHEN age(query_start) > ''24h'' THEN ''Day DD Mon'' ELSE ''HH24:MI:SS'' END),E'' started\n'' + ,CASE WHEN wait_event IS NOT NULL THEN ''waiting'' ELSE state END,E''\n'' + ,date_trunc(''second'',age(now(),query_start)),'' ago'' + ) AS lock_state + FROM blocking_lock blocker + JOIN waiting_lock blocked + ON (blocked.blocker_pid = blocker.pid) + LEFT JOIN pg_stat_activity act ON (act.pid = blocked.pid) + LEFT JOIN acquired_lock acq ON (acq.pid = blocked.pid) + WHERE blocker.depth < 5 + ) +SELECT concat(lpad(''=> '', 4*depth, '' ''),pid::text) AS "PID" +, lock_info AS "Lock Info" +, lock_state AS "State" +FROM blocking_lock +ORDER BY seq; + + +-- From https://wiki.postgresql.org/wiki/Index_Maintenance + +CREATE VIEW duplicate_indexes AS SELECT pg_size_pretty(sum(pg_relation_size(idx))::bigint) as size, + (array_agg(idx))[1] as idx1, (array_agg(idx))[2] as idx2, + (array_agg(idx))[3] as idx3, (array_agg(idx))[4] as idx4 +FROM ( + SELECT indexrelid::regclass as idx, (indrelid::text ||E''\n''|| indclass::text ||E''\n''|| indkey::text ||E''\n''|| + coalesce(indexprs::text,'''')||E''\n'' || coalesce(indpred::text,'''')) as key + FROM pg_index) sub +GROUP BY key HAVING count(*)>1 +ORDER BY sum(pg_relation_size(idx)) DESC; + +-- From https://wiki.postgresql.org/wiki/Disk_Usage + +CREATE VIEW table_sizes AS WITH RECURSIVE pg_inherit(inhrelid, inhparent) AS + (select inhrelid, inhparent + FROM pg_inherits + UNION + SELECT child.inhrelid, parent.inhparent + FROM pg_inherit child, pg_inherits parent + WHERE child.inhparent = parent.inhrelid), +pg_inherit_short AS (SELECT * FROM pg_inherit WHERE inhparent NOT IN (SELECT inhrelid FROM pg_inherit)) +SELECT table_schema + , TABLE_NAME + , row_estimate + , pg_size_pretty(total_bytes) AS total + , pg_size_pretty(index_bytes) AS INDEX + , pg_size_pretty(toast_bytes) AS toast + , pg_size_pretty(table_bytes) AS TABLE + , total_bytes::float8 / sum(total_bytes) OVER () AS total_size_share + FROM ( + SELECT *, total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes + FROM ( + SELECT c.oid + , nspname AS table_schema + , relname AS TABLE_NAME + , SUM(c.reltuples) OVER (partition BY parent) AS row_estimate + , SUM(pg_total_relation_size(c.oid)) OVER (partition BY parent) AS total_bytes + , SUM(pg_indexes_size(c.oid)) OVER (partition BY parent) AS index_bytes + , SUM(pg_total_relation_size(reltoastrelid)) OVER (partition BY parent) AS toast_bytes + , parent + FROM ( + SELECT pg_class.oid + , reltuples + , relname + , relnamespace + , pg_class.reltoastrelid + , COALESCE(inhparent, pg_class.oid) parent + FROM pg_class + LEFT JOIN pg_inherit_short ON inhrelid = oid + WHERE relkind IN (''r'', ''p'') + ) c + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace + ) a + WHERE oid = parent +) a +ORDER BY total_bytes DESC; + +-- From: https://wiki.postgresql.org/wiki/Index_Maintenance + +CREATE VIEW index_usage AS SELECT + t.schemaname, + t.tablename, + c.reltuples::bigint AS num_rows, + pg_size_pretty(pg_relation_size(c.oid)) AS table_size, + psai.indexrelname AS index_name, + pg_size_pretty(pg_relation_size(i.indexrelid)) AS index_size, + CASE WHEN i.indisunique THEN ''Y'' ELSE ''N'' END AS "unique", + psai.idx_scan AS number_of_scans, + psai.idx_tup_read AS tuples_read, + psai.idx_tup_fetch AS tuples_fetched +FROM + pg_tables t + LEFT JOIN pg_class c ON t.tablename = c.relname + LEFT JOIN pg_index i ON c.oid = i.indrelid + LEFT JOIN pg_stat_all_indexes psai ON i.indexrelid = psai.indexrelid +WHERE + t.schemaname NOT IN (''pg_catalog'', ''information_schema'') +ORDER BY 1, 2; + +-- From: https://blog.devgenius.io/top-useful-sql-queries-for-postgresql-35ff3355d265 + +CREATE VIEW database_sizes AS SELECT pg_database.datname, + pg_size_pretty(pg_database_size(pg_database.datname)) AS size +FROM pg_database +ORDER BY pg_database_size(pg_database.datname) DESC; + +CREATE VIEW schema_sizes AS SELECT A.schemaname, + pg_size_pretty (SUM(pg_relation_size(C.oid))) as table, + pg_size_pretty (SUM(pg_total_relation_size(C.oid)-pg_relation_size(C.oid))) as index, + pg_size_pretty (SUM(pg_total_relation_size(C.oid))) as table_index, + SUM(n_live_tup) +FROM pg_class C +LEFT JOIN pg_namespace N ON (N.oid = C .relnamespace) +INNER JOIN pg_stat_user_tables A ON C.relname = A.relname +WHERE nspname NOT IN (''pg_catalog'', ''information_schema'') +AND C .relkind <> ''i'' +AND nspname !~ ''^pg_toast'' +GROUP BY A.schemaname; + +CREATE VIEW last_vacuum_analyze AS SELECT relname, + last_vacuum, + last_autovacuum + n_mod_since_analyze, + last_analyze, + last_autoanalyze, + analyze_count, + autoanalyze_count + FROM pg_stat_user_tables; + +CREATE VIEW table_row_estimates AS SELECT + schemaname, + relname, + n_live_tup +FROM + pg_stat_user_tables +ORDER BY + n_live_tup DESC; + +-- postgres-meta queries as views from: https://github.com/supabase/postgres-meta + +CREATE VIEW pgmeta_columns AS SELECT + c.oid :: int8 AS table_id, + nc.nspname AS schema, + c.relname AS table, + (c.oid || ''.'' || a.attnum) AS id, + a.attnum AS ordinal_position, + a.attname AS name, + CASE + WHEN a.atthasdef THEN pg_get_expr(ad.adbin, ad.adrelid) + ELSE NULL + END AS default_value, + CASE + WHEN t.typtype = ''d'' THEN CASE + WHEN bt.typelem <> 0 :: oid + AND bt.typlen = -1 THEN ''ARRAY'' + WHEN nbt.nspname = ''pg_catalog'' THEN format_type(t.typbasetype, NULL) + ELSE ''USER-DEFINED'' + END + ELSE CASE + WHEN t.typelem <> 0 :: oid + AND t.typlen = -1 THEN ''ARRAY'' + WHEN nt.nspname = ''pg_catalog'' THEN format_type(a.atttypid, NULL) + ELSE ''USER-DEFINED'' + END + END AS data_type, + COALESCE(bt.typname, t.typname) AS format, + a.attidentity IN (''a'', ''d'') AS is_identity, + CASE + a.attidentity + WHEN ''a'' THEN ''ALWAYS'' + WHEN ''d'' THEN ''BY DEFAULT'' + ELSE NULL + END AS identity_generation, + a.attgenerated IN (''s'') AS is_generated, + NOT ( + a.attnotnull + OR t.typtype = ''d'' AND t.typnotnull + ) AS is_nullable, + ( + c.relkind IN (''r'', ''p'') + OR c.relkind IN (''v'', ''f'') AND pg_column_is_updatable(c.oid, a.attnum, FALSE) + ) AS is_updatable, + uniques.table_id IS NOT NULL AS is_unique, + array_to_json( + array( + SELECT + enumlabel + FROM + pg_catalog.pg_enum enums + WHERE + enums.enumtypid = coalesce(bt.oid, t.oid) + OR enums.enumtypid = coalesce(bt.typelem, t.typelem) + ORDER BY + enums.enumsortorder + ) + ) AS enums, + col_description(c.oid, a.attnum) AS comment +FROM + pg_attribute a + LEFT JOIN pg_attrdef ad ON a.attrelid = ad.adrelid + AND a.attnum = ad.adnum + JOIN ( + pg_class c + JOIN pg_namespace nc ON c.relnamespace = nc.oid + ) ON a.attrelid = c.oid + JOIN ( + pg_type t + JOIN pg_namespace nt ON t.typnamespace = nt.oid + ) ON a.atttypid = t.oid + LEFT JOIN ( + pg_type bt + JOIN pg_namespace nbt ON bt.typnamespace = nbt.oid + ) ON t.typtype = ''d'' + AND t.typbasetype = bt.oid + LEFT JOIN ( + SELECT + conrelid AS table_id, + conkey[1] AS ordinal_position + FROM pg_catalog.pg_constraint + WHERE contype = ''u'' AND cardinality(conkey) = 1 + ) AS uniques ON uniques.table_id = c.oid AND uniques.ordinal_position = a.attnum +WHERE + NOT pg_is_other_temp_schema(nc.oid) + AND a.attnum > 0 + AND NOT a.attisdropped + AND (c.relkind IN (''r'', ''v'', ''m'', ''f'', ''p'')) + AND ( + pg_has_role(c.relowner, ''USAGE'') + OR has_column_privilege( + c.oid, + a.attnum, + ''SELECT, INSERT, UPDATE, REFERENCES'' + ) + ); + +CREATE VIEW pgmeta_config AS SELECT + name, + setting, + category, + TRIM(split_part(category, ''/'', 1)) AS group, + TRIM(split_part(category, ''/'', 2)) AS subgroup, + unit, + short_desc, + extra_desc, + context, + vartype, + source, + min_val, + max_val, + enumvals, + boot_val, + reset_val, + sourcefile, + sourceline, + pending_restart +FROM + pg_settings +ORDER BY + category, + name; + +CREATE VIEW pgmeta_extensions AS SELECT + e.name, + n.nspname AS schema, + e.default_version, + x.extversion AS installed_version, + e.comment +FROM + pg_available_extensions() e(name, default_version, comment) + LEFT JOIN pg_extension x ON e.name = x.extname + LEFT JOIN pg_namespace n ON x.extnamespace = n.oid; + +CREATE VIEW pgmeta_foreign_tables AS SELECT + c.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS name, + obj_description(c.oid) AS comment +FROM + pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE + c.relkind = ''f''; + +CREATE VIEW pgmeta_functions AS with functions as ( + select + *, + -- proargmodes is null when all arg modes are IN + coalesce( + p.proargmodes, + array_fill(''i''::text, array[cardinality(coalesce(p.proallargtypes, p.proargtypes))]) + ) as arg_modes, + -- proargnames is null when all args are unnamed + coalesce( + p.proargnames, + array_fill(''''::text, array[cardinality(coalesce(p.proallargtypes, p.proargtypes))]) + ) as arg_names, + -- proallargtypes is null when all arg modes are IN + coalesce(p.proallargtypes, p.proargtypes) as arg_types, + array_cat( + array_fill(false, array[pronargs - pronargdefaults]), + array_fill(true, array[pronargdefaults])) as arg_has_defaults + from + pg_proc as p + where + p.prokind = ''f'' +) +select + f.oid::int8 as id, + n.nspname as schema, + f.proname as name, + l.lanname as language, + case + when l.lanname = ''internal'' then '''' + else f.prosrc + end as definition, + case + when l.lanname = ''internal'' then f.prosrc + else pg_get_functiondef(f.oid) + end as complete_statement, + coalesce(f_args.args, ''[]'') as args, + pg_get_function_arguments(f.oid) as argument_types, + pg_get_function_identity_arguments(f.oid) as identity_argument_types, + f.prorettype::int8 as return_type_id, + pg_get_function_result(f.oid) as return_type, + nullif(rt.typrelid::int8, 0) as return_type_relation_id, + f.proretset as is_set_returning_function, + case + when f.provolatile = ''i'' then ''IMMUTABLE'' + when f.provolatile = ''s'' then ''STABLE'' + when f.provolatile = ''v'' then ''VOLATILE'' + end as behavior, + f.prosecdef as security_definer, + f_config.config_params as config_params +from + functions f + left join pg_namespace n on f.pronamespace = n.oid + left join pg_language l on f.prolang = l.oid + left join pg_type rt on rt.oid = f.prorettype + left join ( + select + oid, + jsonb_object_agg(param, value) filter (where param is not null) as config_params + from + ( + select + oid, + (string_to_array(unnest(proconfig), ''=''))[1] as param, + (string_to_array(unnest(proconfig), ''=''))[2] as value + from + functions + ) as t + group by + oid + ) f_config on f_config.oid = f.oid + left join ( + select + oid, + jsonb_agg(jsonb_build_object( + ''mode'', t2.mode, + ''name'', name, + ''type_id'', type_id, + ''has_default'', has_default + )) as args + from + ( + select + oid, + unnest(arg_modes) as mode, + unnest(arg_names) as name, + unnest(arg_types)::int8 as type_id, + unnest(arg_has_defaults) as has_default + from + functions + ) as t1, + lateral ( + select + case + when t1.mode = ''i'' then ''in'' + when t1.mode = ''o'' then ''out'' + when t1.mode = ''b'' then ''inout'' + when t1.mode = ''v'' then ''variadic'' + else ''table'' + end as mode + ) as t2 + group by + t1.oid + ) f_args on f_args.oid = f.oid; + +CREATE VIEW pgmeta_materialized_views AS select + c.oid::int8 as id, + n.nspname as schema, + c.relname as name, + c.relispopulated as is_populated, + obj_description(c.oid) as comment +from + pg_class c + join pg_namespace n on n.oid = c.relnamespace +where + c.relkind = ''m''; + +CREATE VIEW pgmeta_policies AS SELECT + pol.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS table, + c.oid :: int8 AS table_id, + pol.polname AS name, + CASE + WHEN pol.polpermissive THEN ''PERMISSIVE'' :: text + ELSE ''RESTRICTIVE'' :: text + END AS action, + CASE + WHEN pol.polroles = ''{0}'' :: oid [] THEN array_to_json( + string_to_array(''public'' :: text, '''' :: text) :: name [] + ) + ELSE array_to_json( + ARRAY( + SELECT + pg_roles.rolname + FROM + pg_roles + WHERE + pg_roles.oid = ANY (pol.polroles) + ORDER BY + pg_roles.rolname + ) + ) + END AS roles, + CASE + pol.polcmd + WHEN ''r'' :: "char" THEN ''SELECT'' :: text + WHEN ''a'' :: "char" THEN ''INSERT'' :: text + WHEN ''w'' :: "char" THEN ''UPDATE'' :: text + WHEN ''d'' :: "char" THEN ''DELETE'' :: text + WHEN ''*'' :: "char" THEN ''ALL'' :: text + ELSE NULL :: text + END AS command, + pg_get_expr(pol.polqual, pol.polrelid) AS definition, + pg_get_expr(pol.polwithcheck, pol.polrelid) AS check +FROM + pg_policy pol + JOIN pg_class c ON c.oid = pol.polrelid + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace; + +CREATE VIEW pgmeta_primary_keys AS SELECT + n.nspname AS schema, + c.relname AS table_name, + a.attname AS name, + c.oid :: int8 AS table_id +FROM + pg_index i, + pg_class c, + pg_attribute a, + pg_namespace n +WHERE + i.indrelid = c.oid + AND c.relnamespace = n.oid + AND a.attrelid = c.oid + AND a.attnum = ANY (i.indkey) + AND i.indisprimary; + +CREATE VIEW pgmeta_publications AS SELECT + p.oid :: int8 AS id, + p.pubname AS name, + p.pubowner::regrole::text AS owner, + p.pubinsert AS publish_insert, + p.pubupdate AS publish_update, + p.pubdelete AS publish_delete, + p.pubtruncate AS publish_truncate, + CASE + WHEN p.puballtables THEN NULL + ELSE pr.tables + END AS tables +FROM + pg_catalog.pg_publication AS p + LEFT JOIN LATERAL ( + SELECT + COALESCE( + array_agg( + json_build_object( + ''id'', + c.oid :: int8, + ''name'', + c.relname, + ''schema'', + nc.nspname + ) + ), + ''{}'' + ) AS tables + FROM + pg_catalog.pg_publication_rel AS pr + JOIN pg_class AS c ON pr.prrelid = c.oid + join pg_namespace as nc on c.relnamespace = nc.oid + WHERE + pr.prpubid = p.oid + ) AS pr ON 1 = 1; + +CREATE VIEW pgmeta_relationships AS SELECT + c.oid :: int8 AS id, + c.conname AS constraint_name, + nsa.nspname AS source_schema, + csa.relname AS source_table_name, + sa.attname AS source_column_name, + nta.nspname AS target_table_schema, + cta.relname AS target_table_name, + ta.attname AS target_column_name +FROM + pg_constraint c + JOIN ( + pg_attribute sa + JOIN pg_class csa ON sa.attrelid = csa.oid + JOIN pg_namespace nsa ON csa.relnamespace = nsa.oid + ) ON sa.attrelid = c.conrelid + AND sa.attnum = ANY (c.conkey) + JOIN ( + pg_attribute ta + JOIN pg_class cta ON ta.attrelid = cta.oid + JOIN pg_namespace nta ON cta.relnamespace = nta.oid + ) ON ta.attrelid = c.confrelid + AND ta.attnum = ANY (c.confkey) +WHERE + c.contype = ''f''; + +CREATE VIEW pgmeta_roles AS -- TODO: Consider using pg_authid vs. pg_roles for unencrypted password field +SELECT + oid :: int8 AS id, + rolname AS name, + rolsuper AS is_superuser, + rolcreatedb AS can_create_db, + rolcreaterole AS can_create_role, + rolinherit AS inherit_role, + rolcanlogin AS can_login, + rolreplication AS is_replication_role, + rolbypassrls AS can_bypass_rls, + ( + SELECT + COUNT(*) + FROM + pg_stat_activity + WHERE + pg_roles.rolname = pg_stat_activity.usename + ) AS active_connections, + CASE WHEN rolconnlimit = -1 THEN current_setting(''max_connections'') :: int8 + ELSE rolconnlimit + END AS connection_limit, + rolpassword AS password, + rolvaliduntil AS valid_until, + rolconfig AS config +FROM + pg_roles; + +CREATE VIEW pgmeta_schemas AS select + n.oid::int8 as id, + n.nspname as name, + u.rolname as owner +from + pg_namespace n, + pg_roles u +where + n.nspowner = u.oid + and ( + pg_has_role(n.nspowner, ''USAGE'') + or has_schema_privilege(n.oid, ''CREATE, USAGE'') + ) + and not pg_catalog.starts_with(n.nspname, ''pg_temp_'') + and not pg_catalog.starts_with(n.nspname, ''pg_toast_temp_''); + +CREATE VIEW pgmeta_tables AS SELECT + c.oid :: int8 AS id, + nc.nspname AS schema, + c.relname AS name, + c.relrowsecurity AS rls_enabled, + c.relforcerowsecurity AS rls_forced, + CASE + WHEN c.relreplident = ''d'' THEN ''DEFAULT'' + WHEN c.relreplident = ''i'' THEN ''INDEX'' + WHEN c.relreplident = ''f'' THEN ''FULL'' + ELSE ''NOTHING'' + END AS replica_identity, + pg_total_relation_size(format(''%I.%I'', nc.nspname, c.relname)) :: int8 AS bytes, + pg_size_pretty( + pg_total_relation_size(format(''%I.%I'', nc.nspname, c.relname)) + ) AS size, + pg_stat_get_live_tuples(c.oid) AS live_rows_estimate, + pg_stat_get_dead_tuples(c.oid) AS dead_rows_estimate, + obj_description(c.oid) AS comment +FROM + pg_namespace nc + JOIN pg_class c ON nc.oid = c.relnamespace +WHERE + c.relkind IN (''r'', ''p'') + AND NOT pg_is_other_temp_schema(nc.oid) + AND ( + pg_has_role(c.relowner, ''USAGE'') + OR has_table_privilege( + c.oid, + ''SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER'' + ) + OR has_any_column_privilege(c.oid, ''SELECT, INSERT, UPDATE, REFERENCES'') + ); + + +CREATE VIEW pgmeta_triggers AS SELECT + pg_t.oid AS id, + pg_t.tgrelid AS table_id, + CASE + WHEN pg_t.tgenabled = ''D'' THEN ''DISABLED'' + WHEN pg_t.tgenabled = ''O'' THEN ''ORIGIN'' + WHEN pg_t.tgenabled = ''R'' THEN ''REPLICA'' + WHEN pg_t.tgenabled = ''A'' THEN ''ALWAYS'' + END AS enabled_mode, + ( + STRING_TO_ARRAY( + ENCODE(pg_t.tgargs, ''escape''), ''\000'' + ) + )[:pg_t.tgnargs] AS function_args, + is_t.trigger_name AS name, + is_t.event_object_table AS table, + is_t.event_object_schema AS schema, + is_t.action_condition AS condition, + is_t.action_orientation AS orientation, + is_t.action_timing AS activation, + ARRAY_AGG(is_t.event_manipulation)::text[] AS events, + pg_p.proname AS function_name, + pg_n.nspname AS function_schema +FROM + pg_trigger AS pg_t +JOIN + pg_class AS pg_c +ON pg_t.tgrelid = pg_c.oid +JOIN information_schema.triggers AS is_t +ON is_t.trigger_name = pg_t.tgname +AND pg_c.relname = is_t.event_object_table +JOIN pg_proc AS pg_p +ON pg_t.tgfoid = pg_p.oid +JOIN pg_namespace AS pg_n +ON pg_p.pronamespace = pg_n.oid +GROUP BY + pg_t.oid, + pg_t.tgrelid, + pg_t.tgenabled, + pg_t.tgargs, + pg_t.tgnargs, + is_t.trigger_name, + is_t.event_object_table, + is_t.event_object_schema, + is_t.action_condition, + is_t.action_orientation, + is_t.action_timing, + pg_p.proname, + pg_n.nspname; + + +CREATE VIEW pgmeta_types AS select + t.oid::int8 as id, + t.typname as name, + n.nspname as schema, + format_type (t.oid, null) as format, + coalesce(t_enums.enums, ''[]'') as enums, + coalesce(t_attributes.attributes, ''[]'') as attributes, + obj_description (t.oid, ''pg_type'') as comment +from + pg_type t + left join pg_namespace n on n.oid = t.typnamespace + left join ( + select + enumtypid, + jsonb_agg(enumlabel order by enumsortorder) as enums + from + pg_enum + group by + enumtypid + ) as t_enums on t_enums.enumtypid = t.oid + left join ( + select + oid, + jsonb_agg( + jsonb_build_object(''name'', a.attname, ''type_id'', a.atttypid::int8) + order by a.attnum asc + ) as attributes + from + pg_class c + join pg_attribute a on a.attrelid = c.oid + where + c.relkind = ''c'' and not a.attisdropped + group by + c.oid + ) as t_attributes on t_attributes.oid = t.typrelid +where + ( + t.typrelid = 0 + or ( + select + c.relkind = ''c'' + from + pg_class c + where + c.oid = t.typrelid + ) + ); + +CREATE VIEW pgmeta_version AS SELECT + version(), + current_setting(''server_version_num'') :: int8 AS version_number, + ( + SELECT + COUNT(*) AS active_connections + FROM + pg_stat_activity + ) AS active_connections, + current_setting(''max_connections'') :: int8 AS max_connections; + +CREATE VIEW pgmeta_views AS SELECT + c.oid :: int8 AS id, + n.nspname AS schema, + c.relname AS name, + -- See definition of information_schema.views + (pg_relation_is_updatable(c.oid, false) & 20) = 20 AS is_updatable, + obj_description(c.oid) AS comment +FROM + pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE + c.relkind = ''v''; +', + ' +# michelp@adminpack + +A Trusted Language Extension containing a variety of useful databse admin queries. + +Postgres database admins are often faced with a variety of issues that +require them introspecting the database state. This extension +contains a mixed-bag of views that reflect useful information for +admins. + +## Installation + +Using dbdev: + +``` +postgres=> select dbdev.install(''michelp@adminpack''); + install +--------- + t +(1 row) + +postgres=> create schema adminpack; +CREATE SCHEMA +postgres=> create extension "michelp@adminpack" with schema adminpack; +CREATE EXTENSION +``` + +This will the extension into the `adminpack` schema, substitute with +some other schema if you want it to go somewhere else. + +## Index Bloat + +Index updates and querying can end up getting slower and slower over +time due to what''s called "index bloat". This is where frequent +updates and deletes can cause space in index data structures to go +unused. Understanding when this is happening is not obvious, so there +is a rather complex query you can run to detect it. + +`index_bloat` + +| Column | Type | +|------------------|------------------| +| current_database | name | +| schemaname | name | +| tblname | name | +| idxname | name | +| real_size | numeric | +| extra_size | numeric | +| extra_pct | double precision | +| fillfactor | integer | +| bloat_size | double precision | +| bloat_pct | double precision | +| is_na | boolean | + + +## Table Bloat + +Like index bloat, tables with high update and delete rates can also +end up containing lots of allocated but unused space. This query +shows which tables have the most bloat. + +`table_bloat` + +| Column | Type | +|------------------|------------------| +| current_database | name | +| schemaname | name | +| tblname | name | +| real_size | numeric | +| extra_size | double precision | +| extra_pct | double precision | +| fillfactor | integer | +| bloat_size | double precision | +| bloat_pct | double precision | +| is_na | boolean | + + +## Blocking PID Tree + +Postgres queries can block each other, and this blocking relationship +can form a tree, where A blocks B, which blocks C, and so on. This +query formats blocking queries into a tree structure so it''s easy to +see what query is causing the blockage. + +`blocking_pid_tree` + +| Column | Type | +|-----------|------| +| PID | text | +| Lock Info | text | +| State | text | + + +## Duplicate Indexes + +If a table contains duplicate indexes, then unecessary work is done +updating and storing them, this query will show up to 4 duplicate +indexes per table if they exist. + +`duplicate_indexes` + +| Column | Type | +|--------|----------| +| size | text | +| idx1 | regclass | +| idx2 | regclass | +| idx3 | regclass | +| idx4 | regclass | + + +## Table Sizes + +This view shows tables and their sizes. + +`table_sizes` + +| Column | Type | +|------------------|------------------| +| table_schema | name | +| table_name | name | +| row_estimate | real | +| total | text | +| index | text | +| toast | text | +| table | text | +| total_size_share | double precision | + + +## Schema Sizes + +This view shows schemas and their sizes, which is the sum of the sizes +of all the tables and indexes in the schema. + +`schema_sizes` + +| Column | Type | +|-------------|---------| +| schemaname | name | +| table | text | +| index | text | +| table_index | text | +| sum | numeric | + + +## Index Usage + +This view shows index size and usage. An unused index still needs to +be updated and that takes time an storage, so it''s a good candidate to +drop. + +`index_usage` + +| Column | Type | +|-----------------|--------| +| schemaname | name | +| tablename | name | +| num_rows | bigint | +| table_size | text | +| index_name | name | +| index_size | text | +| unique | text | +| number_of_scans | bigint | +| tuples_read | bigint | +| tuples_fetched | bigint | + + +## Last Vacuum Analyze + +This views shows the last time a table was vacuumed an analyzed. + +`last_vacuum_analyze` + +| Column | Type | +|---------------------|--------------------------| +| relname | name | +| last_vacuum | timestamp with time zone | +| n_mod_since_analyze | timestamp with time zone | +| last_analyze | timestamp with time zone | +| last_autoanalyze | timestamp with time zone | +| analyze_count | bigint | +| autoanalyze_count | bigint | + +## Table Row Estimates + +This view shows estimates for the number of rows in a table. This is +just an estimate and depends on up to date table statistics. + +`table_row_estimates` + +| Column | Type | +|------------|--------| +| schemaname | name | +| relname | name | +| n_live_tup | bigint | + +## PGMeta Columns + +This view shows infromation for the columns of tables in the system. + +`pgmeta_columns` + +| Column | Type | +|---------------------|----------| +| table_id | bigint | +| schema | name | +| table | name | +| id | text | +| ordinal_position | smallint | +| name | name | +| default_value | text | +| data_type | text | +| format | name | +| is_identity | boolean | +| identity_generation | text | +| is_generated | boolean | +| is_nullable | boolean | +| is_updatable | boolean | +| is_unique | boolean | +| enums | json | +| comment | text | + +## PGMeta Config + +This views shows the configuration of the database. + +`pgmeta_config` + +| Column | Type | +|-----------------|---------| +| name | text | +| setting | text | +| category | text | +| group | text | +| subgroup | text | +| unit | text | +| short_desc | text | +| extra_desc | text | +| context | text | +| vartype | text | +| source | text | +| min_val | text | +| max_val | text | +| enumvals | text[] | +| boot_val | text | +| reset_val | text | +| sourcefile | text | +| sourceline | integer | +| pending_restart | boolean | + +## PGMeta Extensions + +This view shows installed extensions in the database. + +`pgmeta_extensions` + +| Column | Type | +|-------------------|------| +| name | name | +| schema | name | +| default_version | text | +| installed_version | text | +| comment | text | + +## PGMeta Foreign Tables + +This view shows foreign tables in the database. + +`pgmeta_foreign_tables` + +| Column | Type | +|---------|--------| +| id | bigint | +| schema | name | +| name | name | +| comment | text | + +## PGMeta Functions + +This view shows functions in the database. + +`pgmeta_functions` + +| Column | Type | +|---------------------------|---------| +| id | bigint | +| schema | name | +| name | name | +| language | name | +| definition | text | +| complete_statement | text | +| args | jsonb | +| argument_types | text | +| identity_argument_types | text | +| return_type_id | bigint | +| return_type | text | +| return_type_relation_id | bigint | +| is_set_returning_function | boolean | +| behavior | text | +| security_definer | boolean | +| config_params | jsonb | + +## PGMeta Materialized Views + +This view shows materialized views in the database. + +`pgmeta_materialized_views` + +| Column | Type | +|--------------|---------| +| id | bigint | +| schema | name | +| name | name | +| is_populated | boolean | +| comment | text | + +## PGMeta Policies + +This view shows Row Level Security Policies in the database. + +`pgmeta_policies` + +| Column | Type | +|------------|--------| +| id | bigint | +| schema | name | +| table | name | +| table_id | bigint | +| name | name | +| action | text | +| roles | json | +| command | text | +| definition | text | +| check | text | + +## PGMeta Primary Keys + +This view shows primary keys in the database. + +`pgmeta_primary_keys` + +| Column | Type | +|------------|--------| +| schema | name | +| table_name | name | +| name | name | +| table_id | bigint | + +## PGMeta Publications + +This view shows logical replication publishers in the database. + +`pgmeta_publications` + +| Column | Type | +|------------------|---------| +| id | bigint | +| name | name | +| owner | text | +| publish_insert | boolean | +| publish_update | boolean | +| publish_delete | boolean | +| publish_truncate | boolean | +| tables | json[] | + +## PGMeta Relationships + +This view shows foreign key relationships in the database. + +`pgmeta_relationships` + +| Column | Type | +|---------------------|--------| +| id | bigint | +| constraint_name | name | +| source_schema | name | +| source_table_name | name | +| source_column_name | name | +| target_table_schema | name | +| target_table_name | name | +| target_column_name | name | + +## PGMeta Roles + +This view shows roles in the database system. Note that roles are +global objects and apply to all databases. + +`pgmeta_roles` + +| Column | Type | +|---------------------|--------------------------| +| id | bigint | +| name | name | +| is_superuser | boolean | +| can_create_db | boolean | +| can_create_role | boolean | +| inherit_role | boolean | +| can_login | boolean | +| is_replication_role | boolean | +| can_bypass_rls | boolean | +| active_connections | bigint | +| connection_limit | bigint | +| password | text | +| valid_until | timestamp with time zone | +| config | text[] | + +## PGMeta Schemas + +This view shows all schemas in the database. + +`pgmeta_schemas` + +| Column | Type | +|--------|--------| +| id | bigint | +| name | name | +| owner | name | + +## PGMeta Tables + +This view shows all tables in the database. + +`pgmeta_tables` + +| Column | Type | +|--------------------|---------| +| id | bigint | +| schema | name | +| name | name | +| rls_enabled | boolean | +| rls_forced | boolean | +| replica_identity | text | +| bytes | bigint | +| size | text | +| live_rows_estimate | bigint | +| dead_rows_estimate | bigint | +| comment | text | + +## PGMeta Triggers + +This view shows all triggers in the database. + +`pgmeta_triggers` + +| Column | Type | +|-----------------|-----------------------------------| +| id | oid | +| table_id | oid | +| enabled_mode | text | +| function_args | text[] | +| name | information_schema.sql_identifier | +| table | information_schema.sql_identifier | +| schema | information_schema.sql_identifier | +| condition | information_schema.character_data | +| orientation | information_schema.character_data | +| activation | information_schema.character_data | +| events | text[] | +| function_name | name | +| function_schema | name | + +## PGMeta Types + +This view shows all types in the database. + +`pgmeta_types` + +| Column | Type | +|------------|--------| +| id | bigint | +| name | name | +| schema | name | +| format | text | +| enums | jsonb | +| attributes | jsonb | +| comment | text | + +## PGMeta Version + +This view shows the current database version. + +`pgmeta_version` + +| Column | Type | +|--------------------|--------| +| version | text | +| version_number | bigint | +| active_connections | bigint | +| max_connections | bigint | + +## PGMeta Views + +This view shows all views in the database. + +`pgmeta_views` + +| Column | Type | +|--------------|---------| +| id | bigint | +| schema | name | +| name | name | +| is_updatable | boolean | +| comment | text | +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113329_olirice@index_advisor-0.2.1_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113329_olirice@index_advisor-0.2.1_100.snap new file mode 100644 index 000000000..366590e41 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113329_olirice@index_advisor-0.2.1_100.snap @@ -0,0 +1,354 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231207113329_olirice@index_advisor-0.2.1.sql +--- +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_alias = 'olirice@index_advisor' + ), + (0, 2, 1), + ' + +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + hypopg_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''hypopg'' + and installed_version is not null + ); + begin + + if not hypopg_exists then + raise + exception ''"olirice@index_advisor" requires "hypopg"'' + using hint = ''Run "create extension hypopg" and try again''; + end if; + end +$$; + +create or replace function index_advisor( + query text +) + returns table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[], + errors text[] + ) + volatile + language plpgsql + as $$ +declare + n_args int; + prepared_statement_name text = ''index_advisor_working_statement''; + hypopg_schema_name text = (select extnamespace::regnamespace::text from pg_extension where extname = ''hypopg''); + explain_plan_statement text; + error_message text; + rec record; + plan_initial jsonb; + plan_final jsonb; + statements text[] = ''{}''; +begin + + -- Remove comment lines (its common that they contain semicolons) + query := trim( + regexp_replace( + regexp_replace( + regexp_replace(query,''\/\*.+\*\/'', '''', ''g''), + ''--[^\r\n]*'', '' '', ''g''), + ''\s+'', '' '', ''g'') + ); + + -- Remove trailing semicolon + query := regexp_replace(query, '';\s*$'', ''''); + + begin + -- Disallow multiple statements + if query ilike ''%;%'' then + raise exception ''Query must not contain a semicolon''; + end if; + + -- Hack to support PostgREST because the prepared statement for args incorrectly defaults to text + query := replace(query, ''WITH pgrst_payload AS (SELECT $1 AS json_data)'', ''WITH pgrst_payload AS (SELECT $1::json AS json_data)''); + + -- Create a prepared statement for the given query + deallocate all; + execute format(''prepare %I as %s'', prepared_statement_name, query); + + -- Detect how many arguments are present in the prepared statement + n_args = ( + select + coalesce(array_length(parameter_types, 1), 0) + from + pg_prepared_statements + where + name = prepared_statement_name + limit + 1 + ); + + -- Create a SQL statement that can be executed to collect the explain plan + explain_plan_statement = format( + ''set local plan_cache_mode = force_generic_plan; explain (format json) execute %I%s'', + --''explain (format json) execute %I%s'', + prepared_statement_name, + case + when n_args = 0 then '''' + else format( + ''(%s)'', array_to_string(array_fill(''null''::text, array[n_args]), '','') + ) + end + ); + + -- Store the query plan before any new indexes + execute explain_plan_statement into plan_initial; + + -- Create possible indexes + for rec in ( + with extension_regclass as ( + select + distinct objid as oid + from + pg_catalog.pg_depend + where + deptype = ''e'' + ) + select + pc.relnamespace::regnamespace::text as schema_name, + pc.relname as table_name, + pa.attname as column_name, + format( + ''select %I.hypopg_create_index($i$create index on %I.%I(%I)$i$)'', + hypopg_schema_name, + pc.relnamespace::regnamespace::text, + pc.relname, + pa.attname + ) hypopg_statement + from + pg_catalog.pg_class pc + join pg_catalog.pg_attribute pa + on pc.oid = pa.attrelid + left join extension_regclass er + on pc.oid = er.oid + left join pg_catalog.pg_index pi + on pc.oid = pi.indrelid + and (select array_agg(x) from unnest(pi.indkey) v(x)) = array[pa.attnum] + and pi.indexprs is null -- ignore expression indexes + and pi.indpred is null -- ignore partial indexes + where + pc.relnamespace::regnamespace::text not in ( -- ignore schema list + ''pg_catalog'', ''pg_toast'', ''information_schema'' + ) + and er.oid is null -- ignore entities owned by extensions + and pc.relkind in (''r'', ''m'') -- regular tables, and materialized views + and pc.relpersistence = ''p'' -- permanent tables (not unlogged or temporary) + and pa.attnum > 0 + and not pa.attisdropped + and pi.indrelid is null + and pa.atttypid in (20,16,1082,1184,1114,701,23,21,700,1083,2950,1700,25,18,1042,1043) + ) + loop + -- Create the hypothetical index + execute rec.hypopg_statement; + end loop; + + -- Create a prepared statement for the given query + -- The original prepared statement MUST be dropped because its plan is cached + execute format(''deallocate %I'', prepared_statement_name); + execute format(''prepare %I as %s'', prepared_statement_name, query); + + -- Store the query plan after new indexes + execute explain_plan_statement into plan_final; + + --raise notice ''%'', plan_final; + + -- Idenfity referenced indexes in new plan + execute format( + ''select + coalesce(array_agg(hypopg_get_indexdef(indexrelid) order by indrelid, indkey::text), $i${}$i$::text[]) + from + %I.hypopg() + where + %s ilike ($i$%%$i$ || indexname || $i$%%$i$) + '', + hypopg_schema_name, + quote_literal(plan_final)::text + ) into statements; + + -- Reset all hypothetical indexes + perform hypopg_reset(); + + -- Reset prepared statements + deallocate all; + + return query values ( + (plan_initial -> 0 -> ''Plan'' -> ''Startup Cost''), + (plan_final -> 0 -> ''Plan'' -> ''Startup Cost''), + (plan_initial -> 0 -> ''Plan'' -> ''Total Cost''), + (plan_final -> 0 -> ''Plan'' -> ''Total Cost''), + statements::text[], + array[]::text[] + ); + return; + + exception when others then + get stacked diagnostics error_message = MESSAGE_TEXT; + + return query values ( + null::jsonb, + null::jsonb, + null::jsonb, + null::jsonb, + array[]::text[], + array[error_message]::text[] + ); + return; + end; + +end; +$$; + +', + ' + +# index_advisor + +`index_advisor` is an extension for recommending indexes to improve query performance. + +## Installation + +Note: + +`hypopg` is a dependency of index_advisor. +Dependency resolution is currently under development. +In the future it will not be necessary to manually create dependencies. + + +```sql +select dbdev.install(''olirice@index_advisor''); +create extension if not exists hypopg; +create extension "olirice@index_advisor" version ''0.2.0''; +``` + +Features: +- Supports generic parameters e.g. `$1`, `$2` +- Supports materialized views +- Identifies tables/columns obfuscaed by views + + +## API + +#### Description +For a given *query*, searches for a set of SQL DDL `create index` statements that improve the query''s execution time; + +#### Signature +```sql +index_advisor(query text) +returns + table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[], + errors text[] + ) +``` + +## Usage + +For a minimal example, the `index_advisor` function can be given a single table query with a filter on an unindexed column. + +```sql +create extension if not exists index_advisor cascade; + +create table book( + id int primary key, + title text not null +); + +select + * +from + index_advisor(''select book.id from book where title = $1''); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements | errors +---------------------+--------------------+-------------------+------------------+-----------------------------------------------------+-------- + 0.00 | 1.17 | 25.88 | 6.40 | {"CREATE INDEX ON public.book USING btree (title)"},| {} +(1 row) +``` + +More complex queries may generate additional suggested indexes + +```sql +create extension if not exists index_advisor cascade; + +create table author( + id serial primary key, + name text not null +); + +create table publisher( + id serial primary key, + name text not null, + corporate_address text +); + +create table book( + id serial primary key, + author_id int not null references author(id), + publisher_id int not null references publisher(id), + title text +); + +create table review( + id serial primary key, + book_id int references book(id), + body text not null +); + +select + * +from + index_advisor('' + select + book.id, + book.title, + publisher.name as publisher_name, + author.name as author_name, + review.body review_body + from + book + join publisher + on book.publisher_id = publisher.id + join author + on book.author_id = author.id + join review + on book.id = review.book_id + where + author.id = $1 + and publisher.id = $2 + ''); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements | errors +---------------------+--------------------+-------------------+------------------+-----------------------------------------------------------+-------- + 27.26 | 12.77 | 68.48 | 42.37 | {"CREATE INDEX ON public.book USING btree (author_id)", | {} + "CREATE INDEX ON public.book USING btree (publisher_id)", + "CREATE INDEX ON public.review USING btree (book_id)"} +(3 rows) +``` +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113329_olirice@index_advisor-0.2.1_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113329_olirice@index_advisor-0.2.1_80.snap new file mode 100644 index 000000000..933c7ce38 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113329_olirice@index_advisor-0.2.1_80.snap @@ -0,0 +1,359 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231207113329_olirice@index_advisor-0.2.1.sql +--- +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_alias = 'olirice@index_advisor' + ), + (0, 2, 1), + ' + +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + hypopg_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''hypopg'' + and installed_version is not null + ); + begin + + if not hypopg_exists then + raise + exception ''"olirice@index_advisor" requires "hypopg"'' + using hint = ''Run "create extension hypopg" and try again''; + end if; + end +$$; + +create or replace function index_advisor( + query text +) + returns table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[], + errors text[] + ) + volatile + language plpgsql + as $$ +declare + n_args int; + prepared_statement_name text = ''index_advisor_working_statement''; + hypopg_schema_name text = (select extnamespace::regnamespace::text from pg_extension where extname = ''hypopg''); + explain_plan_statement text; + error_message text; + rec record; + plan_initial jsonb; + plan_final jsonb; + statements text[] = ''{}''; +begin + + -- Remove comment lines (its common that they contain semicolons) + query := trim( + regexp_replace( + regexp_replace( + regexp_replace(query,''\/\*.+\*\/'', '''', ''g''), + ''--[^\r\n]*'', '' '', ''g''), + ''\s+'', '' '', ''g'') + ); + + -- Remove trailing semicolon + query := regexp_replace(query, '';\s*$'', ''''); + + begin + -- Disallow multiple statements + if query ilike ''%;%'' then + raise exception ''Query must not contain a semicolon''; + end if; + + -- Hack to support PostgREST because the prepared statement for args incorrectly defaults to text + query := replace(query, ''WITH pgrst_payload AS (SELECT $1 AS json_data)'', ''WITH pgrst_payload AS (SELECT $1::json AS json_data)''); + + -- Create a prepared statement for the given query + deallocate all; + execute format(''prepare %I as %s'', prepared_statement_name, query); + + -- Detect how many arguments are present in the prepared statement + n_args = ( + select + coalesce(array_length(parameter_types, 1), 0) + from + pg_prepared_statements + where + name = prepared_statement_name + limit + 1 + ); + + -- Create a SQL statement that can be executed to collect the explain plan + explain_plan_statement = format( + ''set local plan_cache_mode = force_generic_plan; explain (format json) execute %I%s'', + --''explain (format json) execute %I%s'', + prepared_statement_name, + case + when n_args = 0 then '''' + else format( + ''(%s)'', array_to_string(array_fill(''null''::text, array[n_args]), '','') + ) + end + ); + + -- Store the query plan before any new indexes + execute explain_plan_statement into plan_initial; + + -- Create possible indexes + for rec in ( + with extension_regclass as ( + select + distinct objid as oid + from + pg_catalog.pg_depend + where + deptype = ''e'' + ) + select + pc.relnamespace::regnamespace::text as schema_name, + pc.relname as table_name, + pa.attname as column_name, + format( + ''select %I.hypopg_create_index($i$create index on %I.%I(%I)$i$)'', + hypopg_schema_name, + pc.relnamespace::regnamespace::text, + pc.relname, + pa.attname + ) hypopg_statement + from + pg_catalog.pg_class pc + join pg_catalog.pg_attribute pa + on pc.oid = pa.attrelid + left join extension_regclass er + on pc.oid = er.oid + left join pg_catalog.pg_index pi + on pc.oid = pi.indrelid + and (select array_agg(x) from unnest(pi.indkey) v(x)) = array[pa.attnum] + and pi.indexprs is null -- ignore expression indexes + and pi.indpred is null -- ignore partial indexes + where + pc.relnamespace::regnamespace::text not in ( -- ignore schema list + ''pg_catalog'', ''pg_toast'', ''information_schema'' + ) + and er.oid is null -- ignore entities owned by extensions + and pc.relkind in (''r'', ''m'') -- regular tables, and materialized views + and pc.relpersistence = ''p'' -- permanent tables (not unlogged or temporary) + and pa.attnum > 0 + and not pa.attisdropped + and pi.indrelid is null + and pa.atttypid in (20,16,1082,1184,1114,701,23,21,700,1083,2950,1700,25,18,1042,1043) + ) + loop + -- Create the hypothetical index + execute rec.hypopg_statement; + end loop; + + -- Create a prepared statement for the given query + -- The original prepared statement MUST be dropped because its plan is cached + execute format(''deallocate %I'', prepared_statement_name); + execute format(''prepare %I as %s'', prepared_statement_name, query); + + -- Store the query plan after new indexes + execute explain_plan_statement into plan_final; + + --raise notice ''%'', plan_final; + + -- Idenfity referenced indexes in new plan + execute format( + ''select + coalesce(array_agg(hypopg_get_indexdef(indexrelid) order by indrelid, indkey::text), $i${}$i$::text[]) + from + %I.hypopg() + where + %s ilike ($i$%%$i$ || indexname || $i$%%$i$) + '', + hypopg_schema_name, + quote_literal(plan_final)::text + ) into statements; + + -- Reset all hypothetical indexes + perform hypopg_reset(); + + -- Reset prepared statements + deallocate all; + + return query values ( + (plan_initial -> 0 -> ''Plan'' -> ''Startup Cost''), + (plan_final -> 0 -> ''Plan'' -> ''Startup Cost''), + (plan_initial -> 0 -> ''Plan'' -> ''Total Cost''), + (plan_final -> 0 -> ''Plan'' -> ''Total Cost''), + statements::text[], + array[]::text[] + ); + return; + + exception when others then + get stacked diagnostics error_message = MESSAGE_TEXT; + + return query values ( + null::jsonb, + null::jsonb, + null::jsonb, + null::jsonb, + array[]::text[], + array[error_message]::text[] + ); + return; + end; + +end; +$$; + +', + ' + +# index_advisor + +`index_advisor` is an extension for recommending indexes to improve query performance. + +## Installation + +Note: + +`hypopg` is a dependency of index_advisor. +Dependency resolution is currently under development. +In the future it will not be necessary to manually create dependencies. + + +```sql +select dbdev.install(''olirice@index_advisor''); +create extension if not exists hypopg; +create extension "olirice@index_advisor" version ''0.2.0''; +``` + +Features: +- Supports generic parameters e.g. `$1`, `$2` +- Supports materialized views +- Identifies tables/columns obfuscaed by views + + +## API + +#### Description +For a given *query*, searches for a set of SQL DDL `create index` statements that improve the query''s execution time; + +#### Signature +```sql +index_advisor(query text) +returns + table ( + startup_cost_before jsonb, + startup_cost_after jsonb, + total_cost_before jsonb, + total_cost_after jsonb, + index_statements text[], + errors text[] + ) +``` + +## Usage + +For a minimal example, the `index_advisor` function can be given a single table query with a filter on an unindexed column. + +```sql +create extension if not exists index_advisor cascade; + +create table book( + id int primary key, + title text not null +); + +select + * +from + index_advisor(''select book.id from book where title = $1''); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements | errors +---------------------+--------------------+-------------------+------------------+-----------------------------------------------------+-------- + 0.00 | 1.17 | 25.88 | 6.40 | {"CREATE INDEX ON public.book USING btree (title)"},| {} +(1 row) +``` + +More complex queries may generate additional suggested indexes + +```sql +create extension if not exists index_advisor cascade; + +create table author( + id serial primary key, + name text not null +); + +create table publisher( + id serial primary key, + name text not null, + corporate_address text +); + +create table book( + id serial primary key, + author_id int not null references author(id), + publisher_id int not null references publisher(id), + title text +); + +create table review( + id serial primary key, + book_id int references book(id), + body text not null +); + +select + * +from + index_advisor('' + select + book.id, + book.title, + publisher.name as publisher_name, + author.name as author_name, + review.body review_body + from + book + join publisher + on book.publisher_id = publisher.id + join author + on book.author_id = author.id + join review + on book.id = review.book_id + where + author.id = $1 + and publisher.id = $2 + ''); + + startup_cost_before | startup_cost_after | total_cost_before | total_cost_after | index_statements | errors +---------------------+--------------------+-------------------+------------------+-----------------------------------------------------------+-------- + 27.26 | 12.77 | 68.48 | 42.37 | {"CREATE INDEX ON public.book USING btree (author_id)", | {} + "CREATE INDEX ON public.book USING btree (publisher_id)", + "CREATE INDEX ON public.review USING btree (book_id)"} +(3 rows) +``` +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113857_olirice@read_once-0.3.2_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113857_olirice@read_once-0.3.2_100.snap new file mode 100644 index 000000000..9b00b7000 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113857_olirice@read_once-0.3.2_100.snap @@ -0,0 +1,156 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231207113857_olirice@read_once-0.3.2.sql +--- +insert into app.package_versions (package_id, version_struct, sql, description_md) +values + ( + ( + select + id + from + app.packages + where + package_alias = 'olirice@read_once' + ), + (0, 3, 2), + ' + +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + pg_cron_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''pg_cron'' + and installed_version is not null + ); + begin + + if not pg_cron_exists then + raise + exception ''"olirice@read_once" requires "pg_cron"'' + using hint = ''Run "create extension pg_cron" and try again''; + end if; + end +$$; + + +create schema read_once; + +create unlogged table read_once.messages( + id uuid primary key default gen_random_uuid(), + contents text not null default '''', + created_at timestamp default now() +); + +revoke all on read_once.messages from public; +revoke usage on schema read_once from public; + +create or replace function send_message( + contents text +) + returns uuid + security definer + volatile + strict + language sql + as +$$ + insert into read_once.messages(contents) + values ($1) + returning id; +$$; + +create or replace function read_message(id uuid) + returns text + security definer + volatile + strict + language sql + as +$$ + delete from read_once.messages + where read_once.messages.id = $1 + returning contents; +$$ +', + ' + +# read_once + +A Supabase application for sending messages that can only be read once + +Features: +- messages can only be read one time +- messages are not logged in PostgreSQL write-ahead-log (WAL) + +## Installation + +`pg_cron` is a dependency of `read_once`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + +To expose the `send_message` and `read_message` functions over HTTP, install the extension in a schema that is on the search_path. + + +For example: +```sql +select dbdev.install(''olirice@read_once''); +create extension if not exists pg_cron; +create extension "olirice@read_once" + schema public + version ''0.3.1''; +``` + + +## HTTP API + +### Create a Message + +```sh +curl -X POST https://.supabase.co/rest/v1/rpc/send_message \ + -H ''apiKey: '' \ + -H ''Content-Type: application/json'' + --data-raw ''{"contents": "hello, dbdev!"} + +# Returns: "2989156b-2356-4543-9d1b-19dfb8ec3268" +``` + +### Read a Message + +```sh +curl -X https://.supabase.co/rest/v1/rpc/read_message + -H ''apiKey: '' \ + -H ''Content-Type: application/json'' \ + --data-raw ''{"id": "2989156b-2356-4543-9d1b-19dfb8ec3268"} + +# Returns: "hello, dbdev!" +``` + + +## SQL API + +### Create a Message + +```sql +-- Creates a new messages and returns its unique id +create or replace function send_message( + contents text +) + returns uuid +``` + +### Read a Message + +```sql +-- Read a message by its id +create or replace function read_message( + id uuid +) + returns text +``` +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113857_olirice@read_once-0.3.2_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113857_olirice@read_once-0.3.2_80.snap new file mode 100644 index 000000000..031b97ca3 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20231207113857_olirice@read_once-0.3.2_80.snap @@ -0,0 +1,161 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20231207113857_olirice@read_once-0.3.2.sql +--- +insert into app.package_versions ( + package_id, + version_struct, + sql, + description_md +) +values + ( + ( + select + id + from + app.packages + where + package_alias = 'olirice@read_once' + ), + (0, 3, 2), + ' + +-- Enforce requirements +-- Workaround to https://github.com/aws/pg_tle/issues/183 +do $$ + declare + pg_cron_exists boolean = exists( + select 1 + from pg_available_extensions + where + name = ''pg_cron'' + and installed_version is not null + ); + begin + + if not pg_cron_exists then + raise + exception ''"olirice@read_once" requires "pg_cron"'' + using hint = ''Run "create extension pg_cron" and try again''; + end if; + end +$$; + + +create schema read_once; + +create unlogged table read_once.messages( + id uuid primary key default gen_random_uuid(), + contents text not null default '''', + created_at timestamp default now() +); + +revoke all on read_once.messages from public; +revoke usage on schema read_once from public; + +create or replace function send_message( + contents text +) + returns uuid + security definer + volatile + strict + language sql + as +$$ + insert into read_once.messages(contents) + values ($1) + returning id; +$$; + +create or replace function read_message(id uuid) + returns text + security definer + volatile + strict + language sql + as +$$ + delete from read_once.messages + where read_once.messages.id = $1 + returning contents; +$$ +', + ' + +# read_once + +A Supabase application for sending messages that can only be read once + +Features: +- messages can only be read one time +- messages are not logged in PostgreSQL write-ahead-log (WAL) + +## Installation + +`pg_cron` is a dependency of `read_once`. +Dependency resolution is currently under development. +In the near future it will not be necessary to manually create dependencies. + +To expose the `send_message` and `read_message` functions over HTTP, install the extension in a schema that is on the search_path. + + +For example: +```sql +select dbdev.install(''olirice@read_once''); +create extension if not exists pg_cron; +create extension "olirice@read_once" + schema public + version ''0.3.1''; +``` + + +## HTTP API + +### Create a Message + +```sh +curl -X POST https://.supabase.co/rest/v1/rpc/send_message \ + -H ''apiKey: '' \ + -H ''Content-Type: application/json'' + --data-raw ''{"contents": "hello, dbdev!"} + +# Returns: "2989156b-2356-4543-9d1b-19dfb8ec3268" +``` + +### Read a Message + +```sh +curl -X https://.supabase.co/rest/v1/rpc/read_message + -H ''apiKey: '' \ + -H ''Content-Type: application/json'' \ + --data-raw ''{"id": "2989156b-2356-4543-9d1b-19dfb8ec3268"} + +# Returns: "hello, dbdev!" +``` + + +## SQL API + +### Create a Message + +```sql +-- Creates a new messages and returns its unique id +create or replace function send_message( + contents text +) + returns uuid +``` + +### Read a Message + +```sql +-- Read a message by its id +create or replace function read_message( + id uuid +) + returns text +``` +' + ); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240108072747_update_provider_id_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240108072747_update_provider_id_100.snap new file mode 100644 index 000000000..fc838e9ab --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240108072747_update_provider_id_100.snap @@ -0,0 +1,5 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20240108072747_update_provider_id.sql +--- +update auth.identities set provider_id = email where provider = 'email'; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240108072747_update_provider_id_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240108072747_update_provider_id_80.snap new file mode 100644 index 000000000..fc838e9ab --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240108072747_update_provider_id_80.snap @@ -0,0 +1,5 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20240108072747_update_provider_id.sql +--- +update auth.identities set provider_id = email where provider = 'email'; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240605122023_fix_view_permissions_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240605122023_fix_view_permissions_100.snap new file mode 100644 index 000000000..ea3902c80 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240605122023_fix_view_permissions_100.snap @@ -0,0 +1,33 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20240605122023_fix_view_permissions.sql +--- +alter view public.packages + set (security_invoker = 'true'); + +alter view public.package_versions + set (security_invoker = 'true'); + +alter view public.package_upgrades + set (security_invoker = 'true'); + +create policy packages_select_policy_anon +on app.packages +as permissive +for select +to anon +using (true); + +create policy package_versions_select_policy_anon +on app.package_versions +as permissive +for select +to anon +using (true); + +create policy package_upgrades_select_policy_anon +on app.package_upgrades +as permissive +for select +to anon +using (true); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240605122023_fix_view_permissions_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240605122023_fix_view_permissions_80.snap new file mode 100644 index 000000000..ea3902c80 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240605122023_fix_view_permissions_80.snap @@ -0,0 +1,33 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20240605122023_fix_view_permissions.sql +--- +alter view public.packages + set (security_invoker = 'true'); + +alter view public.package_versions + set (security_invoker = 'true'); + +alter view public.package_upgrades + set (security_invoker = 'true'); + +create policy packages_select_policy_anon +on app.packages +as permissive +for select +to anon +using (true); + +create policy package_versions_select_policy_anon +on app.package_versions +as permissive +for select +to anon +using (true); + +create policy package_upgrades_select_policy_anon +on app.package_upgrades +as permissive +for select +to anon +using (true); diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240705083738_remove_contact_email_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240705083738_remove_contact_email_100.snap new file mode 100644 index 000000000..7dd751914 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240705083738_remove_contact_email_100.snap @@ -0,0 +1,37 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20240705083738_remove_contact_email.sql +--- +drop view if exists public.accounts; + +create view public.accounts + with (security_invoker = 'true') +as select + acc.id, + acc.handle, + obj.name as avatar_path, + acc.display_name, + acc.bio, + acc.created_at +from + app.accounts as acc + left outer join + storage.objects as obj + on acc.avatar_id = obj.id; + +drop view if exists public.organizations; + +create view public.organizations + with (security_invoker = 'true') +as select + org.id, + org.handle, + obj.name as avatar_path, + org.display_name, + org.bio, + org.created_at +from + app.organizations as org + left outer join + storage.objects as obj + on org.avatar_id = obj.id; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240705083738_remove_contact_email_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240705083738_remove_contact_email_80.snap new file mode 100644 index 000000000..7dd751914 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20240705083738_remove_contact_email_80.snap @@ -0,0 +1,37 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20240705083738_remove_contact_email.sql +--- +drop view if exists public.accounts; + +create view public.accounts + with (security_invoker = 'true') +as select + acc.id, + acc.handle, + obj.name as avatar_path, + acc.display_name, + acc.bio, + acc.created_at +from + app.accounts as acc + left outer join + storage.objects as obj + on acc.avatar_id = obj.id; + +drop view if exists public.organizations; + +create view public.organizations + with (security_invoker = 'true') +as select + org.id, + org.handle, + obj.name as avatar_path, + org.display_name, + org.bio, + org.created_at +from + app.organizations as org + left outer join + storage.objects as obj + on org.avatar_id = obj.id; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250106073735_jwt_secret_from_vault_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250106073735_jwt_secret_from_vault_100.snap new file mode 100644 index 000000000..5040ab259 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250106073735_jwt_secret_from_vault_100.snap @@ -0,0 +1,65 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20250106073735_jwt_secret_from_vault.sql +--- +create or replace function public.redeem_access_token(access_token text) +returns text +language plpgsql +strict +security definer +as $function$ +declare + token_id uuid; + token bytea; + tokens_row app.user_id_and_token_hash; + token_valid boolean; + now timestamp; + one_hour_from_now timestamp; + issued_at int; + expiry_at int; + jwt_secret text; +begin + -- validate access token + if length(access_token) != 64 then + raise exception 'Invalid token'; + end if; + + if substring(access_token from 1 for 4) != 'dbd_' then + raise exception 'Invalid token'; + end if; + + token_id := substring(access_token from 5 for 32)::uuid; + token := app.base64url_decode(substring(access_token from 37)); + + select t.user_id, t.token_hash + into tokens_row + from app.access_tokens t + where t.id = token_id; + + -- TODO: do a constant time comparison + if tokens_row.token_hash != sha256(token) then + raise exception 'Invalid token'; + end if; + + -- Generate JWT token + now := current_timestamp; + one_hour_from_now := now + interval '1 hour'; + issued_at := date_part('epoch', now); + expiry_at := date_part('epoch', one_hour_from_now); + + -- read the jwt secret from vault + select decrypted_secret + into jwt_secret + from vault.decrypted_secrets + where name = 'app.jwt_secret'; + + return sign(json_build_object( + 'aud', 'authenticated', + 'role', 'authenticated', + 'iss', 'database.dev', + 'sub', tokens_row.user_id, + 'iat', issued_at, + 'exp', expiry_at + ), jwt_secret); +end; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250106073735_jwt_secret_from_vault_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250106073735_jwt_secret_from_vault_80.snap new file mode 100644 index 000000000..5040ab259 --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250106073735_jwt_secret_from_vault_80.snap @@ -0,0 +1,65 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20250106073735_jwt_secret_from_vault.sql +--- +create or replace function public.redeem_access_token(access_token text) +returns text +language plpgsql +strict +security definer +as $function$ +declare + token_id uuid; + token bytea; + tokens_row app.user_id_and_token_hash; + token_valid boolean; + now timestamp; + one_hour_from_now timestamp; + issued_at int; + expiry_at int; + jwt_secret text; +begin + -- validate access token + if length(access_token) != 64 then + raise exception 'Invalid token'; + end if; + + if substring(access_token from 1 for 4) != 'dbd_' then + raise exception 'Invalid token'; + end if; + + token_id := substring(access_token from 5 for 32)::uuid; + token := app.base64url_decode(substring(access_token from 37)); + + select t.user_id, t.token_hash + into tokens_row + from app.access_tokens t + where t.id = token_id; + + -- TODO: do a constant time comparison + if tokens_row.token_hash != sha256(token) then + raise exception 'Invalid token'; + end if; + + -- Generate JWT token + now := current_timestamp; + one_hour_from_now := now + interval '1 hour'; + issued_at := date_part('epoch', now); + expiry_at := date_part('epoch', one_hour_from_now); + + -- read the jwt secret from vault + select decrypted_secret + into jwt_secret + from vault.decrypted_secrets + where name = 'app.jwt_secret'; + + return sign(json_build_object( + 'aud', 'authenticated', + 'role', 'authenticated', + 'iss', 'database.dev', + 'sub', tokens_row.user_id, + 'iat', issued_at, + 'exp', expiry_at + ), jwt_secret); +end; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250217100252_restrict_accounts_and_orgs_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250217100252_restrict_accounts_and_orgs_100.snap new file mode 100644 index 000000000..5255a16ff --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250217100252_restrict_accounts_and_orgs_100.snap @@ -0,0 +1,42 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20250217100252_restrict_accounts_and_orgs.sql +--- +alter policy accounts_select_policy on app.accounts to authenticated using (id = auth.uid()); + +alter policy organizations_select_policy +on app.organizations +to authenticated +using (app.is_organization_maintainer(auth.uid(), id)); + +create or replace function public.get_account(handle text) +returns setof public.accounts +language sql +strict +security definer +as $function$ +select id, handle, avatar_path, display_name, bio, created_at + from public.accounts a + where a.handle = get_account.handle + and auth.uid() is not null; +$function$; + +create or replace function public.get_organization(handle text) +returns setof public.organizations +language sql +strict +security definer +as $function$ +select id, handle, avatar_path, display_name, bio, created_at + from public.organizations o + where o.handle = get_organization.handle + and auth.uid() is not null; +$function$; + +grant SELECT on table app.accounts to service_role; + +grant SELECT on table app.organizations to service_role; + +grant SELECT on table app.packages to service_role; + +grant SELECT on table app.package_versions to service_role; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250217100252_restrict_accounts_and_orgs_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250217100252_restrict_accounts_and_orgs_80.snap new file mode 100644 index 000000000..3f147d5dc --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250217100252_restrict_accounts_and_orgs_80.snap @@ -0,0 +1,48 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20250217100252_restrict_accounts_and_orgs.sql +--- +alter policy accounts_select_policy +on app.accounts +to authenticated +using (id = auth.uid()); + +alter policy organizations_select_policy +on app.organizations +to authenticated +using (app.is_organization_maintainer( + auth.uid(), + id +)); + +create or replace function public.get_account(handle text) +returns setof public.accounts +language sql +strict +security definer +as $function$ +select id, handle, avatar_path, display_name, bio, created_at + from public.accounts a + where a.handle = get_account.handle + and auth.uid() is not null; +$function$; + +create or replace function public.get_organization(handle text) +returns setof public.organizations +language sql +strict +security definer +as $function$ +select id, handle, avatar_path, display_name, bio, created_at + from public.organizations o + where o.handle = get_organization.handle + and auth.uid() is not null; +$function$; + +grant SELECT on table app.accounts to service_role; + +grant SELECT on table app.organizations to service_role; + +grant SELECT on table app.packages to service_role; + +grant SELECT on table app.package_versions to service_role; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250804111152_remove_dbdev_from_popular_packages_100.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250804111152_remove_dbdev_from_popular_packages_100.snap new file mode 100644 index 000000000..86e8452ba --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250804111152_remove_dbdev_from_popular_packages_100.snap @@ -0,0 +1,17 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20250804111152_remove_dbdev_from_popular_packages.sql +--- +create or replace function public.popular_packages() +returns setof public.packages +language sql +stable +as $function$ +select * from public.packages p + where p.package_name != 'supabase-dbdev' + order by ( + select (dm.downloads_30_day * 5) + (dm.downloads_90_days * 2) + dm.downloads_180_days + from public.download_metrics dm + where dm.package_id = p.id + ) desc nulls last, p.created_at desc; +$function$; diff --git a/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250804111152_remove_dbdev_from_popular_packages_80.snap b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250804111152_remove_dbdev_from_popular_packages_80.snap new file mode 100644 index 000000000..86e8452ba --- /dev/null +++ b/crates/pgls_pretty_print/tests/snapshots/multi/tests__20250804111152_remove_dbdev_from_popular_packages_80.snap @@ -0,0 +1,17 @@ +--- +source: crates/pgls_pretty_print/tests/tests.rs +input_file: crates/pgls_pretty_print/tests/data/multi/20250804111152_remove_dbdev_from_popular_packages.sql +--- +create or replace function public.popular_packages() +returns setof public.packages +language sql +stable +as $function$ +select * from public.packages p + where p.package_name != 'supabase-dbdev' + order by ( + select (dm.downloads_30_day * 5) + (dm.downloads_90_days * 2) + dm.downloads_180_days + from public.download_metrics dm + where dm.package_id = p.id + ) desc nulls last, p.created_at desc; +$function$;