diff --git a/CHANGES.rst b/CHANGES.rst index 536e238f0a..1696c87a31 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -24,6 +24,12 @@ esa.emds.einsteinprobe - New module to access the ESA Einstein Probe Science Archive. [#3511] +mast +^^^^ + +- Updated ``Catalogs`` interface for MAST TAP catalogs, including collection/catalog discovery helpers and a unified query workflow across + positional and non-positional searches. [#3582] + API changes @@ -76,6 +82,19 @@ mast - The ``objectname`` keyword is deprecated in ``MastMissions`` in favor of ``object_names``. [#3540] - The ``objectname`` parameter in ``Catalogs``, ``Observations``, ``Tesscut``, and ``utils`` is deprecated in favor of ``object_name``. [#3567] +- ``Catalogs`` has been refactored around VO-TAP queries. The new workflow uses ``collection`` + ``catalog`` (instead of HSC/PanSTARRS-specific + assumptions), supports discovery helpers (``get_collections``, ``get_catalogs``, ``get_column_metadata``), and + adds ``supports_spatial_queries`` to inspect positional-query support before querying. [#3582] +- ``Catalogs.query_criteria`` now provides a unified query interface for positional and non-positional searches, with support for + cone searches around coordinates or an object, STC-S regions via the ``region`` parameter, column selection, sorting, count-only queries, + pagination via ``limit``/``offset``, and a ``filters`` dictionary for column names that conflict with method arguments. [#3582] +- ``Catalogs.query_region`` and ``Catalogs.query_object`` now follow the same unified backend as ``query_criteria``, including support + for ``collection``/``catalog``, ``limit``/``offset``, ``select_cols``, sorting, and advanced criteria filters. [#3582] +- The legacy ``version``, ``pagesize``, and ``page`` parameters in ``Catalogs`` query methods are now deprecated in favor + of ``collection``/``catalog`` and ``limit``/``offset``. [#3582] +- Passing a collection name via ``Catalogs(..., catalog=...)`` is deprecated; use the ``collection`` parameter instead. [#3582] +- ``Catalogs`` legacy HSC-only helper methods (``query_hsc_matchid[_async]``, ``get_hsc_spectra[_async]``, and + ``download_hsc_spectra``) are deprecated and will be removed in a future release. [#3582] vo_conesearch ^^^^^^^^^^^^^ diff --git a/astroquery/mast/__init__.py b/astroquery/mast/__init__.py index fa0b9e8ff8..85c465724b 100644 --- a/astroquery/mast/__init__.py +++ b/astroquery/mast/__init__.py @@ -20,9 +20,6 @@ class Conf(_config.ConfigNamespace): ssoserver = _config.ConfigItem( 'https://ssoportal.stsci.edu', 'MAST SSO Portal server.') - catalogs_server = _config.ConfigItem( - 'https://catalogs.mast.stsci.edu', - 'Catalogs.MAST server.') timeout = _config.ConfigItem( 600, 'Time limit for requests from the STScI server.') @@ -37,18 +34,28 @@ class Conf(_config.ConfigNamespace): conf = Conf() -from .cutouts import TesscutClass, Tesscut, ZcutClass, Zcut, HapcutClass, Hapcut -from .observations import Observations, ObservationsClass, MastClass, Mast +from . import utils from .collections import Catalogs, CatalogsClass +from .cutouts import Hapcut, HapcutClass, Tesscut, TesscutClass, Zcut, ZcutClass from .missions import MastMissions, MastMissionsClass -from . import utils - -__all__ = ['Observations', 'ObservationsClass', - 'Catalogs', 'CatalogsClass', - 'MastMissions', 'MastMissionsClass', - 'Mast', 'MastClass', - 'Tesscut', 'TesscutClass', - 'Zcut', 'ZcutClass', - 'Hapcut', 'HapcutClass', - 'Conf', 'conf', 'utils', - ] +from .observations import Mast, MastClass, Observations, ObservationsClass + +__all__ = [ + "Observations", + "ObservationsClass", + "Catalogs", + "CatalogsClass", + "MastMissions", + "MastMissionsClass", + "Mast", + "MastClass", + "Tesscut", + "TesscutClass", + "Zcut", + "ZcutClass", + "Hapcut", + "HapcutClass", + "Conf", + "conf", + "utils", +] diff --git a/astroquery/mast/catalog_collection.py b/astroquery/mast/catalog_collection.py new file mode 100644 index 0000000000..de452d6300 --- /dev/null +++ b/astroquery/mast/catalog_collection.py @@ -0,0 +1,516 @@ +import difflib +from dataclasses import dataclass +from typing import Dict, Optional + +from astropy.table import Table +from pyvo.dal import DALQueryError, TAPService + +from .. import log +from ..exceptions import InvalidQueryError +from . import conf, utils + +__all__ = ["CatalogCollection"] + +DEFAULT_CATALOGS = { + "caom": "dbo.obspointing", + "gaiadr3": "dbo.gaia_source", + "hsc": "dbo.SumMagAper2CatView", + "hscv2": "dbo.SumMagAper2CatView", + "missionmast": "dbo.hst_science_missionmast", + "ps1dr1": "dbo.MeanObjectView", + "ps1dr2": "dbo.MeanObjectView", + "ps1_dr2": "ps1_dr2.forced_mean_object", + "skymapperdr4": "dr4.master", + "tic": "dbo.CatalogRecord", + "classy": "dbo.targets", + "ullyses": "dbo.sciencemetadata", + "goods": "dbo.goods_master_view", + "3dhst": "dbo.HLSP_3DHST_summary", + "candels": "dbo.candels_master_view", + "deepspace": "dbo.DeepSpace_Summary", + "tic_v82": "tic_v82.source", +} + +GROUPED_COLLECTION_ENDPOINTS = ["mast_catalogs", "roman_catalogs"] + + +@dataclass +class CatalogMetadata: + """ + Data class to hold metadata about a catalog, including column metadata, + RA/Dec column names, and spatial query support. + """ + + column_metadata: Table + ra_column: Optional[str] + dec_column: Optional[str] + supports_spatial_queries: bool + + +class CatalogCollection: + """ + This class provides an interface to interact with MAST catalog collections via TAP service. + """ + + TAP_BASE_URL = conf.server + "/vo-tap/api/v0.1/" + _discovered_collections = None + _collection_parent_map = None + + @classmethod + def discover_collections(cls): + """ + Discover collection names available through TAP and track parent collections. + + Returns + ------- + `~astropy.table.Table` + A table containing collection_name and parent_collection columns. + """ + if cls._discovered_collections is not None: + return cls._discovered_collections + + # Query TAP service for collection names + url = cls.TAP_BASE_URL + "openapi.json" + response = utils._simple_request(url) + response.raise_for_status() + data = response.json() + + try: + collection_enum = data["components"]["schemas"]["CatalogName"]["enum"] + except KeyError: + raise RuntimeError("Failed to discover collections from TAP service: Unexpected response format") + + collection_parent_map = {} + + # Discover collections stored under grouped TAP collections + for parent_collection in GROUPED_COLLECTION_ENDPOINTS: + if parent_collection not in collection_enum: + continue + + tap_service = TAPService(cls.TAP_BASE_URL + parent_collection) + result = tap_service.run_sync("SELECT TOP 5000 table_name FROM tap_schema.tables") + tables = result.to_table() + + for table_name in tables["table_name"]: + table_name = str(table_name) + if table_name.lower().startswith("tap_schema."): + continue + + collection_name = table_name.split(".", 1)[0].lower() + collection_parent_map.setdefault(collection_name, parent_collection) + + # Include standalone collections in map + for collection_name in collection_enum: + normalized_name = collection_name.lower() + if normalized_name in GROUPED_COLLECTION_ENDPOINTS: + continue + collection_parent_map.setdefault(normalized_name, normalized_name) + + collection_names = sorted(collection_parent_map.keys()) + parent_names = [collection_parent_map[name] for name in collection_names] + cls._collection_parent_map = collection_parent_map + cls._discovered_collections = Table( + [collection_names, parent_names], + names=("collection_name", "parent_collection"), + ) + + return cls._discovered_collections + + @classmethod + def get_parent_collection(cls, collection_name): + """ + Return the parent TAP collection for a user-facing collection name. + + Parameters + ---------- + collection_name : str + The user-facing collection name to get the parent collection for. + """ + if not isinstance(collection_name, str): + raise InvalidQueryError(f"Collection name must be a string, got {type(collection_name)}") + + if cls._collection_parent_map is None: + cls.discover_collections() + + normalized_name = collection_name.lower().strip() + parent_collection = cls._collection_parent_map.get(normalized_name) + + if parent_collection is None: + raise InvalidQueryError(f"Collection '{collection_name}' not found") + + return parent_collection + + def __init__(self, collection): + """ + Initialize a CatalogCollection object for interacting with a MAST catalog collection. + + Parameters + ---------- + collection : str + The name of the MAST catalog collection to interact with. + """ + if not isinstance(collection, str): + raise ValueError(f"Collection name must be a string, got {type(collection)}") + + self.name = collection.strip().lower() + self._parent_collection = None + self._tap_service = None + + # Get catalogs within this collection + self._catalogs = None # Lazy-loaded property + + # ADQL functions supported by this collection's TAP service + self._supported_adql_functions = None # Lazy-loaded property + + # Determine the default catalog lazily to avoid requests during initialization + self._default_catalog = None + + # Cache for catalog metadata to avoid redundant queries + self._catalog_metadata_cache: Dict[str, CatalogMetadata] = dict() + + # Cache the catalog lookup mapping for validating catalog names in queries + self._catalog_lookup = None + self._no_prefix_lookup = None + + @property + def parent_collection(self): + if self._parent_collection is None: + self._parent_collection = self.get_parent_collection(self.name) + return self._parent_collection + + @property + def tap_service(self): + if self._tap_service is None: + self._tap_service = TAPService(self.TAP_BASE_URL + self.parent_collection) + return self._tap_service + + @property + def default_catalog(self): + if self._default_catalog is None: + self._default_catalog = self.get_default_catalog() + return self._default_catalog + + @property + def catalogs(self): + if self._catalogs is None: + self._catalogs = self._fetch_catalogs() + return self._catalogs + + @property + def catalog_names(self): + return self.catalogs["catalog_name"].tolist() + + @property + def supported_adql_functions(self): + if self._supported_adql_functions is None: + self._supported_adql_functions = self._fetch_adql_supported_functions() + return self._supported_adql_functions + + def get_catalog_metadata(self, catalog): + """ + For a given catalog, cache and return metadata about its columns and capabilities. + + Parameters + ---------- + catalog : str + The catalog within the collection to get metadata for. + + Returns + ------- + CatalogMetadata + A CatalogMetadata object containing metadata about the specified catalog, including column metadata, + RA/Dec column names, and spatial query support. + """ + # Verify catalog validity for this collection + catalog = self._verify_catalog(catalog) + + # Check cache first + if catalog in self._catalog_metadata_cache: + return self._catalog_metadata_cache[catalog] + + # Get column metadata + metadata = self._get_column_metadata(catalog) + + # Get RA/Dec column names + ra_col, dec_col = self._get_ra_dec_column_names(metadata) + + # Determine if spatial queries are supported + supports_adql_geometry = all(func in self.supported_adql_functions for func in ("POINT", "CIRCLE", "CONTAINS")) + + # Try an inexpensive spatial query if RA/Dec columns are known + supports_spatial_queries = supports_adql_geometry and ra_col is not None and dec_col is not None + if supports_spatial_queries: + # If an ra and dec column exist, test spatial query support + spatial_query = ( + f"SELECT TOP 0 * FROM {catalog} WHERE CONTAINS(POINT('ICRS', {ra_col}, {dec_col}), " + "CIRCLE('ICRS', 0, 0, 0.001)) = 1" + ) + try: + self.tap_service.search(spatial_query) + except DALQueryError: + supports_spatial_queries = False + + meta = CatalogMetadata( + column_metadata=metadata, + ra_column=ra_col, + dec_column=dec_col, + supports_spatial_queries=supports_spatial_queries, + ) + + # Cache and return + self._catalog_metadata_cache[catalog] = meta + return meta + + def get_default_catalog(self): + """ + Get the default catalog for this collection. This is the first catalog that does not start with "tap_schema.". + + Returns + ------- + str + The default catalog name. + """ + # Check if collection has a known default catalog + if self.name in DEFAULT_CATALOGS: + return DEFAULT_CATALOGS[self.name] + + # Pick default catalog = first one that does NOT start with "tap_schema." + default_catalog = next((c for c in self.catalog_names if not c.startswith("tap_schema.")), None) + + # If no valid catalog found, fallback to the first one + if default_catalog is None: + default_catalog = self.catalog_names[0] if self.catalog_names else None + + return default_catalog + + def run_tap_query(self, adql): + """ + Run a TAP query against the specified catalog. + + Parameters + ---------- + adql : str + The ADQL query string. + + Returns + ------- + response : `~astropy.table.Table` + The result of the TAP query as an Astropy Table. + """ + log.debug(f"Running TAP query on collection '{self.name}': {adql}") + try: + result = self.tap_service.run_sync(adql) + except DALQueryError as e: + raise InvalidQueryError(f"TAP query failed for collection '{self.name}': {e}") + return result.to_table() + + def _fetch_catalogs(self): + """ + Retrieve the list of catalogs in this collection. + + Returns + ------- + `~astropy.table.Table` + A table containing the catalog names and descriptions for this collection. + """ + log.debug(f"Fetching available tables for collection '{self.name}' from MAST TAP service.") + query = "SELECT TOP 5000 table_name, description FROM tap_schema.tables" + + # If this catalog is within a grouped collection, filter to only tables that belong to this collection + if self.parent_collection in GROUPED_COLLECTION_ENDPOINTS: + query += f" WHERE table_name LIKE '{self.name}.%'" + + result = self.tap_service.run_sync(query) + + # Rename table_name to catalog_name for clarity + result_table = result.to_table() + result_table.rename_column("table_name", "catalog_name") + + return result_table + + def _fetch_adql_supported_functions(self): + """ + Retrieve the ADQL supported functions of the TAP service. + + Returns + ------- + set + A set of supported ADQL geometry functions (e.g. "POINT", "CIRCLE", "CONTAINS", etc.) for + this collection's TAP service. + """ + adql_functions = ["CIRCLE", "POLYGON", "POINT", "CONTAINS", "INTERSECTS"] + supported = set() + feature_id = "ivo://ivoa.net/std/TAPRegExt#features-adqlgeo" + for capability in self.tap_service.capabilities: + if capability.standardid != "ivo://ivoa.net/std/TAP": + continue + + for lang in capability.languages: + if lang.name != "ADQL": + continue + + for func in adql_functions: + if lang.get_feature(feature_id, func): + supported.add(func) + + return supported + + def _verify_catalog(self, catalog): + """ + Verify that the specified catalog is valid for this collection and return the correct catalog name. + Raises an error if the catalog is not valid. + + Parameters + ---------- + catalog : str + The catalog to be verified. + + Returns + ------- + str + The validated catalog name. + + Raises + ------ + InvalidQueryError + If the specified catalog is not valid for the given collection. + """ + catalog = catalog.lower().strip() + + if self._catalog_lookup is not None and self._no_prefix_lookup is not None: + lookup = self._catalog_lookup + no_prefix_map = self._no_prefix_lookup + else: + # Build a mapping for case-insensitive and no-prefix lookup + lookup = {} + no_prefix_map = {} + for cat in self.catalog_names: + cat_lower = cat.lower() + lookup[cat_lower] = cat # case-insensitive match + no_prefix = cat_lower.split(".")[-1] + if no_prefix not in no_prefix_map: + no_prefix_map[no_prefix] = [cat] # no-prefix match (first occurrence) + else: + no_prefix_map[no_prefix].append(cat) + + # Add unambiguous no-prefix matches to lookup + for no_prefix, cats in no_prefix_map.items(): + if len(cats) == 1: + lookup[no_prefix] = cats[0] + + # Cache the lookup maps for future calls + self._catalog_lookup = lookup + self._no_prefix_lookup = no_prefix_map + + # Direct or unambiguous no-prefix match + if catalog in lookup: + return lookup[catalog] + + # Check for ambiguous no-prefix matches + if catalog in no_prefix_map and len(no_prefix_map[catalog]) > 1: + matches = ", ".join(no_prefix_map[catalog]) + raise InvalidQueryError( + f"Catalog '{catalog}' is ambiguous for collection '{self.name}'. " + f"It matches multiple catalogs: {matches}. Please specify the full catalog name." + ) + + # Suggest closest match (based on full catalog names) + closest = difflib.get_close_matches(catalog, self.catalog_names, n=1) + suggestion = f" Did you mean '{closest[0]}'?" if closest else "" + + raise InvalidQueryError( + f"Catalog '{catalog}' is not recognized for collection '{self.name}'." + f"{suggestion} Available catalogs are: {', '.join(self.catalog_names)}" + ) + + def _get_column_metadata(self, catalog): + """ + For a given catalog, return metadata about its columns. + + Parameters + ---------- + catalog : str + The catalog within the collection to get metadata for. + + Returns + ------- + response : `~astropy.table.Table` + A table containing metadata about the specified table, including column names, data types, and descriptions. + """ + log.debug(f"Fetching column metadata for collection '{self.name}', catalog '{catalog}' from MAST TAP service.") + + query = f""" + SELECT TOP 5000 + column_name, + datatype, + unit, + ucd, + description + FROM tap_schema.columns + WHERE table_name = '{catalog}' + """ + result = self.tap_service.run_sync(query) + + if len(result) == 0: + raise InvalidQueryError(f"Catalog '{catalog}' not found in collection '{self.name}'.") + + return result.to_table() + + def _get_ra_dec_column_names(self, column_metadata): + """ + Return the RA and Dec column names for a given catalog and table. + + Parameters + ---------- + column_metadata : `~astropy.table.Table` + The column metadata table for a catalog. + + Returns + ------- + tuple + A tuple containing the (ra_column, dec_column) names. + """ + # Look for a column with UCD 'pos.eq.ra;meta.main' and 'pos.eq.dec;meta.main' + ra_col = None + dec_col = None + for name, ucd in zip(column_metadata["column_name"], column_metadata["ucd"]): + if ucd and "pos.eq.ra;meta.main" in ucd.lower(): + # TODO: ps1_dr2.mean_object and ps1_dr2.stacked_object has a column that can be used, + # but is not labeled with "meta.main" + ra_col = name + elif ucd and "pos.eq.dec;meta.main" in ucd.lower(): + dec_col = name + return ra_col, dec_col + + def _verify_criteria(self, catalog, **criteria): + """ + Check that criteria keyword arguments are valid column names for the specified collection and catalog. + + Parameters + ---------- + catalog : str + The catalog within the collection to query. + **criteria + Keyword arguments representing criteria filters to apply. + + Raises + ------ + InvalidQueryError + If a keyword does not match any valid column names, an error is raised that suggests the closest + matching column name, if available. + """ + if not criteria: + return + col_names = list(self.get_catalog_metadata(catalog).column_metadata["column_name"]) + col_name_lookup = {col.lower(): col for col in col_names} + + # Check each criteria argument for validity + for kwd in criteria: + if kwd.lower() not in col_name_lookup: + # Suggest closest match for invalid keyword + closest = difflib.get_close_matches(kwd.lower(), list(col_name_lookup.keys()), n=1) + suggestion = f" Did you mean '{col_name_lookup[closest[0]]}'?" if closest else "" + raise InvalidQueryError( + f"Filter '{kwd}' is not recognized for collection '{self.name}' and " + f"catalog '{catalog}'.{suggestion}" + ) diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index ed6baf00f2..f0a7b88dca 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -3,481 +3,687 @@ MAST Collections ================ -This module contains various methods for querying MAST collections such as catalogs. +This module contains methods for discovering and querying MAST catalog collections. """ import difflib -from json import JSONDecodeError -import warnings import os +import re import time +import warnings +from collections.abc import Iterable -from requests import HTTPError, RequestException - -import astropy.units as u import astropy.coordinates as coord -from astropy.table import Table, Row -from astropy.utils.decorators import deprecated_renamed_argument - -from ..utils import commons, async_to_sync +import astropy.units as u +import requests +from astropy.table import Row, Table +from astropy.time import Time +from astropy.utils.decorators import deprecated, deprecated_renamed_argument + +from .. import log +from ..exceptions import InputWarning, InvalidQueryError, NoResultsWarning +from ..utils import async_to_sync from ..utils.class_or_instance import class_or_instance -from ..exceptions import InvalidQueryError, MaxResultsWarning, InputWarning - -from . import utils, conf +from . import conf, utils +from .catalog_collection import CatalogCollection from .core import MastQueryWithLogin +try: + from regions import CircleSkyRegion, PolygonSkyRegion + HAS_REGIONS = True +except ImportError: + HAS_REGIONS = False -__all__ = ['Catalogs', 'CatalogsClass'] +__all__ = ["Catalogs", "CatalogsClass"] @async_to_sync class CatalogsClass(MastQueryWithLogin): """ - MAST catalog query class. - - Class for querying MAST catalog data. + Class for discovering and querying MAST catalog collections. """ - def __init__(self): + TAP_BASE_URL = conf.server + "/vo-tap/api/v0.1/" + + def __init__(self, collection=None, catalog=None): super().__init__() - services = {"panstarrs": {"path": "panstarrs/{data_release}/{table}.json", - "args": {"data_release": "dr2", "table": "mean"}}} - self._catalogs_mast_search_options = ['columns', 'sort_by', 'table', 'data_release'] + self._available_collections = None # Lazy load on first request + self._no_longer_supported_collections = ["ctl", "diskdetective", "galex", "plato"] + self._renamed_collections = {"panstarrs": "ps1_dr2", "gaia": "gaiadr3"} + self._collections_cache = dict() - self._service_api_connection.set_service_params(services, "catalogs", True) + # Default initialization of this class should not trigger network requests + # Only set collection and catalog if explicitly provided, otherwise defer to property setters + # which will handle defaults without network calls + if not collection: + self._collection = CatalogCollection("hsc") # default collection + else: + self.collection = collection # Use the setter for validation if collection is provided - self.catalog_limit = None - self._current_connection = None - self._service_columns = dict() # Info about columns for Catalogs.MAST services + if not catalog: + self._catalog = self._collection.default_catalog + else: + self.catalog = catalog # Use the setter for validation if catalog is provided - def _parse_result(self, response, *, verbose=False): + @property + def collection(self): + """ + The current MAST collection to be queried. + """ + # Return the collection name instead of the object for easier user interaction, + # but keep the object internally for API calls + return self._collection.name - results_table = self._current_connection._parse_result(response, verbose=verbose) + @collection.setter + def collection(self, collection): + """ + Setter that creates a CatalogCollection object when the collection is changed and updates + the catalog accordingly. + """ + collection_obj = self._get_collection_obj(collection) + self._collection = collection_obj - if len(results_table) == self.catalog_limit: - warnings.warn("Maximum catalog results returned, may not include all sources within radius.", - MaxResultsWarning) + # Only change catalog if not set yet or invalid for this collection + if not hasattr(self, "_catalog") or self._catalog not in collection_obj.catalog_names: + self._catalog = collection_obj.default_catalog - return results_table + @property + def catalog(self): + """ + The current catalog within the MAST collection. + """ + return self._catalog - def _get_service_col_config(self, catalog, release='dr2', table='mean'): + @catalog.setter + def catalog(self, catalog): """ - For a given Catalogs.MAST catalog, return a list of all searchable columns and their descriptions. - As of now, this function is exclusive to the Pan-STARRS catalog. + Setter that verifies that the catalog is valid for the current collection. + """ + catalog = self._collection._verify_catalog(catalog) + self._catalog = catalog - Parameters - ---------- - catalog : str - The catalog to be queried. - release : str, optional - Catalog data release to query from. - table : str, optional - Catalog table to query from. + @property + def available_collections(self): + """ + The list of available MAST catalog collections. + """ + if self._available_collections is None: + table = self.get_collections() + self._available_collections = table["collection_name"].tolist() + return self._available_collections + + @class_or_instance + def get_collections(self): + """ + Return a list of available collections from MAST. Returns ------- - response : `~astropy.table.Table` that contains columns names, types, and descriptions + response : `~astropy.table.Table` + A table containing the available MAST collections. """ - # Only supported for PanSTARRS currently - if catalog != 'panstarrs': - return - - service_key = (catalog, release, table) - if service_key not in self._service_columns: - try: - # Send server request to get column list for given parameters - request_url = f'{conf.catalogs_server}/api/v0.1/{catalog}/{release}/{table}/metadata.json' - resp = utils._simple_request(request_url) + # If already cached, use it directly + if getattr(self, "_available_collections", None): + return Table([self._available_collections], names=("collection_name",)) - # Parse JSON and extract necessary info - results = resp.json() + # Otherwise, fetch from TAP service discovery, including grouped collections. + log.debug("Fetching available collections from MAST TAP service.") + collection_table = CatalogCollection.discover_collections()[["collection_name"]] + return collection_table - # Prepare data for Table creation - rows = [] - for result in results: - colname = result.get('column_name') or result.get('name') - dtype = result.get('db_type') - desc = result.get('description', '') + @class_or_instance + def get_catalogs(self, collection=None): + """ + For a given collection, return a list of available catalogs. - if colname is None or dtype is None: - continue # Skip invalid entries + Parameters + ---------- + collection : str, optional + The collection to be queried. - rows.append((colname, dtype, desc)) + Returns + ------- + response : `~astropy.table.Table` + A table containing the available catalogs within the specified collection. + """ + # If no collection specified, use the class attribute + collection_obj = self._get_collection_obj(collection) if collection else self._collection + return collection_obj.catalogs - # Create Table with parsed data - col_table = Table(rows=rows, names=('name', 'data_type', 'description')) - self._service_columns[service_key] = col_table + @class_or_instance + def get_column_metadata(self, collection=None, catalog=None): + """ + For a given collection and catalog, return metadata about the catalog's columns. - except JSONDecodeError as ex: - raise JSONDecodeError(f'Failed to decode JSON response while attempting to get column list' - f' for {catalog} catalog {table}, {release}: {ex}') - except RequestException as ex: - raise ConnectionError(f'Failed to connect to the server while attempting to get column list' - f' for {catalog} catalog {table}, {release}: {ex}') - except Exception as ex: - raise RuntimeError(f'An unexpected error occurred while attempting to get column list' - f' for {catalog} catalog {table}, {release}: {ex}') + Parameters + ---------- + collection : str, optional + The collection to be queried. + catalog : str, optional + The catalog within the collection to get metadata for. - return self._service_columns[service_key] + Returns + ------- + response : `~astropy.table.Table` + A table containing metadata about the specified catalog, including column names, data types, + and descriptions. + """ + collection_obj, catalog = self._parse_inputs(collection, catalog) + return collection_obj._get_column_metadata(catalog) - def _validate_service_criteria(self, catalog, **criteria): + def supports_spatial_queries(self, collection=None, catalog=None): """ - Check that criteria keyword arguments are valid column names for the service. - Raises InvalidQueryError if a criteria argument is invalid. + Check if a given collection and catalog support spatial queries. Parameters ---------- - catalog : str - The catalog to be queried. - **criteria - Keyword arguments representing criteria filters to apply. + collection : str, optional + The collection to be queried. + catalog : str, optional + The catalog within the collection to check. - Raises + Returns ------- - InvalidQueryError - If a keyword does not match any valid column names, an error is raised that suggests the closest - matching column name, if available. - """ - # Ensure that self._service_columns is populated - release = criteria.get('data_release', 'dr2') - table = criteria.get('table', 'mean') - col_config = self._get_service_col_config(catalog, release, table) - - if col_config: - # Check each criteria argument for validity - valid_cols = list(col_config['name']) + self._catalogs_mast_search_options - for kwd in criteria.keys(): - col = next((name for name in valid_cols if name.lower() == kwd.lower()), None) - if not col: - closest_match = difflib.get_close_matches(kwd, valid_cols, n=1) - error_msg = ( - f"Filter '{kwd}' does not exist for {catalog} catalog {table}, {release}. " - f"Did you mean '{closest_match[0]}'?" - if closest_match - else f"Filter '{kwd}' does not exist for {catalog} catalog {table}, {release}." - ) - raise InvalidQueryError(error_msg) + bool + True if the specified catalog supports spatial queries, False otherwise. + """ + collection_obj, catalog = self._parse_inputs(collection, catalog) + return collection_obj.get_catalog_metadata(catalog).supports_spatial_queries @class_or_instance - def query_region_async(self, coordinates, *, radius=0.2*u.deg, catalog="Hsc", - version=None, pagesize=None, page=None, **criteria): + @deprecated_renamed_argument( + "version", + None, + since="0.4.12", + message="The `version` argument is deprecated and " + "will be removed in a future release. Please use `collection` and `catalog` instead.", + ) + @deprecated_renamed_argument( + "pagesize", + None, + since="0.4.12", + message="The `pagesize` argument is deprecated " + "and will be removed in a future release. Please use `limit` instead.", + ) + @deprecated_renamed_argument( + "page", + None, + since="0.4.12", + message="The `page` argument is deprecated " + "and will be removed in a future release. Please use `offset` instead.", + ) + @deprecated_renamed_argument("objectname", "object_name", since="0.4.12") + def query_criteria( + self, + collection=None, + *, + catalog=None, + coordinates=None, + region=None, + object_name=None, + radius=0.2 * u.deg, + resolver=None, + limit=5000, + offset=0, + count_only=False, + select_cols=None, + sort_by=None, + sort_desc=False, + filters=None, + version=None, + pagesize=None, + page=None, + **criteria, + ): """ - Given a sky position and radius, returns a list of catalog entries. - See column documentation for specific catalogs `here `__. + Query a MAST catalog from a given collection using criteria filters. To return columns for a given + collection and catalog, use `~astroquery.mast.CatalogsClass.get_column_metadata`. Parameters ---------- - coordinates : str or `~astropy.coordinates` object - The target around which to search. It may be specified as a - string or as the appropriate `~astropy.coordinates` object. - radius : str or `~astropy.units.Quantity` object, optional - Default 0.2 degrees. - The string must be parsable by `~astropy.coordinates.Angle`. The - appropriate `~astropy.units.Quantity` object from - `~astropy.units` may also be used. Defaults to 0.2 deg. + collection : str, optional + The collection to be queried. If None, uses the instance's `collection` attribute. catalog : str, optional - Default HSC. - The catalog to be queried. - version : int, optional - Version number for catalogs that have versions. Default is highest version. + The catalog within the collection to query. If None, uses the instance's `catalog` attribute. + coordinates : str or `~astropy.coordinates` object, optional + The target around which to search. It may be specified as a string (e.g., '350 -80') or as an + Astropy coordinates object. + region : str | iterable | `~regions.CircleSkyRegion` | `~regions.PolygonSkyRegion`, optional + The region to search within. It may be specified as a STC-S POLYGON or CIRCLE string + (e.g., 'CIRCLE 350 -80 0.2'), an iterable of coordinate pairs, or as an + `~regions.CircleSkyRegion` or `~regions.PolygonSkyRegion`. + object_name : str, optional + The name of the object to resolve and search around. + radius : str or `~astropy.units.Quantity` object, optional + The search radius around the target coordinates or object. Default 0.2 degrees. + resolver : str, optional + The name resolver service to use when resolving ``object_name``. + limit : int, optional + The maximum number of results to return. Default is 5000. + offset : int, optional + The number of rows to skip before starting to return rows. Default is 0. + count_only : bool, optional + If True, only return the count of matching records instead of the records themselves. Default is False. + select_cols : list of str, optional + List of column names to include in the result. If None or empty, all columns are returned. + sort_by : str or list of str, optional + Column name(s) to sort the results by. + sort_desc : bool or list of bool, optional + Indicates whether to sort in descending order for each column in ``sort_by``. If a single bool, + applies to all columns. If a list, must match length of ``sort_by``. Default is False (ascending order). + filters : dict, optional + Another parameter to specify criteria filters as a dictionary. Use this option when the name of a column + conflicts with a named parameter of this method. + version : str, optional + Deprecated. The version argument is no longer used. Please use ``collection`` and ``catalog`` instead. pagesize : int, optional - Default None. - Can be used to override the default pagesize for (set in configs) this query only. - E.g. when using a slow internet connection. + Deprecated. The pagesize argument is no longer used. Please use ``limit`` instead. page : int, optional - Default None. - Can be used to override the default behavior of all results being returned to obtain a - specific page of results. + Deprecated. The page argument is no longer used. Please use ``offset`` instead. **criteria - Other catalog-specific keyword args. - These can be found in the (service documentation)[https://mast.stsci.edu/api/v0/_services.html] - for specific catalogs. For example, one can specify the magtype for an HSC search. - For catalogs available through Catalogs.MAST (PanSTARRS), the Column Name is the keyword, and the argument - should be either an acceptable value for that parameter, or a list consisting values, or tuples of - decorator, value pairs (decorator, value). In addition, columns may be used to select the return columns, - consisting of a list of column names. Results may also be sorted through the query with the parameter - sort_by composed of either a single Column Name to sort ASC, or a list of Column Nmaes to sort ASC or - tuples of Column Name and Direction (ASC, DESC) to indicate sort order (Column Name, DESC). - Detailed information of Catalogs.MAST criteria usage can - be found `here `__. + Keyword arguments representing criteria filters to apply. + + Criteria syntax + ---------------- + - Strings support wildcards using '*' (converted to SQL '%') and '%'. + - Lists are combined with OR for positive values; empty lists yield no matches. + - Numeric columns support comparison operators ('<', '<=', '>', '>=') and inclusive ranges using + the syntax 'low..high' (e.g., '5..10'). Mixed lists of numbers and comparisons are OR-combined. + - Negation: Prefix any value with '!' to negate that predicate. For list inputs, all negated values + for the same column are AND-combined, then ANDed with the OR of the positive values: + (neg1 AND neg2 AND ...) AND (pos1 OR pos2 OR ...). + + Examples + -------- + - file_suffix=['A', 'B', '!C'] -> (file_suffix != 'C') AND (file_suffix IN ('A', 'B')) + - size=['!14400', '<20000'] -> (size != 14400) AND (size < 20000) Returns ------- - response : list of `~requests.Response` + response : `~astropy.table.Table` + A table containing the query results. """ + # Parse pagination params + limit, offset = self._parse_legacy_pagination(limit, offset, pagesize, page) + + # Should not specify both region and coordinates + if coordinates and region: + raise InvalidQueryError("Specify either `region` or `coordinates`, not both.") + + # Should not specify both region and object_name + if object_name and region: + raise InvalidQueryError("Specify either `region` or `object_name`, not both.") + + collection_obj, catalog = self._parse_inputs(collection, catalog) + + # Check for conflicts between named parameters and filters dict + if criteria and filters: + overlap = set(k.lower() for k in criteria) & set(k.lower() for k in filters) + if overlap: + raise InvalidQueryError( + f"Criteria specified both as keyword arguments and in 'filters' for columns: " + f"{', '.join(sorted(overlap))}" + ) - # Put coordinates and radius into consistent format - coordinates = commons.parse_coordinates(coordinates, return_frame='icrs') - - # if radius is just a number we assume degrees - radius = coord.Angle(radius, u.deg) + # Merge criteria from named parameters and filters dict + search_criteria = {} + search_criteria.update(criteria) + if filters: + search_criteria.update(filters) + + # Validate sort_by columns together with criteria keys so all column checks go through one path + validation_criteria = dict(search_criteria) + if sort_by: + sort_by = [sort_by] if isinstance(sort_by, str) else list(sort_by) + validation_criteria.update({col: True for col in sort_by}) + + collection_obj._verify_criteria(catalog, **validation_criteria) + catalog_metadata = collection_obj.get_catalog_metadata(catalog) + column_metadata = catalog_metadata.column_metadata + columns = "*" if not select_cols else self._parse_select_cols(select_cols, column_metadata) + + adql = ( + f"SELECT TOP {limit} {columns} FROM {catalog.lower()} " + if not count_only + else f"SELECT TOP 1 COUNT(*) AS count_all FROM {catalog.lower()} " + ) + has_where = False + if region or coordinates or object_name: + # Check if the catalog supports spatial queries + if not catalog_metadata.supports_spatial_queries: + raise InvalidQueryError( + f"Catalog '{catalog}' in collection '{collection_obj.name}' does not support spatial queries." + ) - # basic params - params = {'ra': coordinates.ra.deg, - 'dec': coordinates.dec.deg, - 'radius': radius.deg} + # Positional query + adql_region = "" + if region: + adql_region = self._create_adql_region(region) + if object_name or coordinates: # Cone search + coordinates = utils.parse_input_location( + coordinates=coordinates, object_name=object_name, resolver=resolver + ) + radius = coord.Angle(radius, u.deg) # If radius is just a number we assume degrees + adql_region = f"CIRCLE('ICRS', {coordinates.ra.deg}, {coordinates.dec.deg}, {radius.to(u.deg).value})" + + region_types = ["POLYGON", "CIRCLE"] + for region_type in region_types: + if region_type in adql_region and region_type not in collection_obj.supported_adql_functions: + raise InvalidQueryError( + f"Catalog '{catalog}' in collection '{collection_obj.name}' " + f"does not support ADQL region type '{region_type}'." + ) - # Determine API connection and service name - if catalog.lower() in self._service_api_connection.SERVICES: - self._current_connection = self._service_api_connection - service = catalog + # Get RA/Dec column names + ra_col = catalog_metadata.ra_column + dec_col = catalog_metadata.dec_column + adql += f"WHERE CONTAINS(POINT('ICRS', {ra_col}, {dec_col}), {adql_region}) = 1 " + has_where = True + + # Add additional constraints + if search_criteria: + conditions = self._format_criteria_conditions(collection_obj, catalog, search_criteria) + if has_where: + adql += "AND " + " AND ".join(conditions) + else: + adql += "WHERE " + " AND ".join(conditions) - # validate user criteria - self._validate_service_criteria(catalog.lower(), **criteria) + # Add sorting if specified + if sort_by: + # Add ORDER BY clause + if isinstance(sort_desc, bool): + sort_desc = [sort_desc] - # adding additional user specified parameters - for prop, value in criteria.items(): - params[prop] = value + if len(sort_desc) not in (1, len(sort_by)): + raise InvalidQueryError("Length of 'sort_desc' must be 1 or equal to length of 'sort_by'.") - else: - self._current_connection = self._portal_api_connection + if len(sort_desc) == 1: + sort_desc = sort_desc * len(sort_by) - # valid criteria keywords - valid_criteria = [] + order_parts = [ + f"{col} {'DESC' if desc else 'ASC'}" + for col, desc in zip(sort_by, sort_desc) + ] - # Sorting out the non-standard portal service names - if catalog.lower() == "hsc": - if version == 2: - service = "Mast.Hsc.Db.v2" - else: - if version not in (3, None): - warnings.warn("Invalid HSC version number, defaulting to v3.", InputWarning) - service = "Mast.Hsc.Db.v3" - - # Hsc specific parameters (can be overridden by user) - self.catalog_limit = criteria.pop('nr', 50000) - valid_criteria = ['nr', 'ni', 'magtype'] - params['nr'] = self.catalog_limit - params['ni'] = criteria.pop('ni', 1) - params['magtype'] = criteria.pop('magtype', 1) - - elif catalog.lower() == "galex": - service = "Mast.Galex.Catalog" - self.catalog_limit = criteria.get('maxrecords', 50000) - - # galex specific parameters (can be overridden by user) - valid_criteria = ['maxrecords'] - params['maxrecords'] = criteria.pop('maxrecords', 50000) - - elif catalog.lower() == "gaia": - if version == 1: - service = "Mast.Catalogs.GaiaDR1.Cone" - else: - if version not in (None, 2): - warnings.warn("Invalid Gaia version number, defaulting to DR2.", InputWarning) - service = "Mast.Catalogs.GaiaDR2.Cone" + adql += " ORDER BY " + ", ".join(order_parts) - elif catalog.lower() == 'plato': - if version in (None, 1): - service = "Mast.Catalogs.Plato.Cone" - else: - warnings.warn("Invalid PLATO catalog version number, defaulting to DR1.", InputWarning) - service = "Mast.Catalogs.Plato.Cone" + # Add offset + if offset: + adql += f" OFFSET {offset}" - else: - service = "Mast.Catalogs." + catalog + ".Cone" - self.catalog_limit = None + # Execute the query + result_table = collection_obj.run_tap_query(adql) - # additional user-specified parameters are not valid - if criteria: - key = next(iter(criteria)) - closest_match = difflib.get_close_matches(key, valid_criteria, n=1) - error_msg = ( - f"Filter '{key}' does not exist for catalog {catalog}. Did you mean '{closest_match[0]}'?" - if closest_match - else f"Filter '{key}' does not exist for catalog {catalog}." - ) - raise InvalidQueryError(error_msg) - - # Parameters will be passed as JSON objects only when accessing the PANSTARRS API - use_json = catalog.lower() == 'panstarrs' + if len(result_table) == 0: + warnings.warn("The query returned no results.", NoResultsWarning) - return self._current_connection.service_request_async(service, params, pagesize=pagesize, page=page, - use_json=use_json) + if count_only: + return result_table["count_all"][0] + else: + # TODO: Add schema browser URL to the result table metadata when available + result_table.meta["collection"] = collection_obj.name + result_table.meta["catalog"] = catalog + return result_table @class_or_instance - @deprecated_renamed_argument('objectname', 'object_name', since='0.4.12') - def query_object_async(self, object_name, *, radius=0.2*u.deg, catalog="Hsc", - pagesize=None, page=None, version=None, resolver=None, **criteria): + @deprecated_renamed_argument( + "version", + None, + since="0.4.12", + message="The `version` argument is deprecated and " + "will be removed in a future release. Please use `collection` and `catalog` instead.", + ) + @deprecated_renamed_argument( + "pagesize", + None, + since="0.4.12", + message="The `pagesize` argument is deprecated " + "and will be removed in a future release. Please use `limit` instead.", + ) + @deprecated_renamed_argument( + "page", + None, + since="0.4.12", + message="The `page` argument is deprecated " + "and will be removed in a future release. Please use `offset` instead.", + ) + def query_region( + self, + coordinates=None, + *, + radius=0.2 * u.deg, + region=None, + collection=None, + catalog=None, + limit=5000, + offset=0, + count_only=False, + select_cols=None, + sort_by=None, + sort_desc=False, + filters=None, + version=None, + pagesize=None, + page=None, + **criteria, + ): """ - Given an object name, returns a list of catalog entries. - See column documentation for specific catalogs `here `__. + Query for MAST catalog entries within a specified region using criteria filters. To return columns for a given + collection and catalog, use `~astroquery.mast.CatalogsClass.get_column_metadata`. Parameters ---------- - object_name : str - The name of the target around which to search. + coordinates : str or `~astropy.coordinates` object, optional + The target around which to search. It may be specified as a string (e.g., '350 -80') or as an + Astropy coordinates object. radius : str or `~astropy.units.Quantity` object, optional - Default 0.2 degrees. - The string must be parsable by `~astropy.coordinates.Angle`. - The appropriate `~astropy.units.Quantity` object from - `~astropy.units` may also be used. Defaults to 0.2 deg. + The search radius around the target coordinates or object. Default 0.2 degrees. + region : str | iterable | `~regions.CircleSkyRegion` | `~regions.PolygonSkyRegion`, optional + The region to search within. It may be specified as a STC-S POLYGON or CIRCLE string + (e.g., 'CIRCLE 350 -80 0.2'), an iterable of coordinate pairs, or as an + `~regions.CircleSkyRegion` or `~regions.PolygonSkyRegion`. + collection : str, optional + The collection to be queried. If None, uses the instance's `collection` attribute. catalog : str, optional - Default HSC. - The catalog to be queried. + The catalog within the collection to query. If None, uses the instance's `catalog` attribute. + limit : int, optional + The maximum number of results to return. Default is 5000. + offset : int, optional + The number of rows to skip before starting to return rows. Default is 0. + count_only : bool, optional + If True, only return the count of matching records instead of the records themselves. Default is False. + select_cols : list of str, optional + List of column names to include in the result. If None or empty, all columns are returned. + sort_by : str or list of str, optional + Column name(s) to sort the results by. + sort_desc : bool or list of bool, optional + Indicates whether to sort in descending order for each column in ``sort_by``. If a single bool, + applies to all columns. If a list, must match length of ``sort_by``. Default is False (ascending order). + filters : dict, optional + Another parameter to specify criteria filters as a dictionary. Use this option when the name of a column + conflicts with a named parameter of this method. + version : str, optional + Deprecated. The version argument is no longer used. Please use ``collection`` and ``catalog`` instead. pagesize : int, optional - Default None. - Can be used to override the default pagesize for (set in configs) this query only. - E.g. when using a slow internet connection. + Deprecated. The pagesize argument is no longer used. Please use ``limit`` instead. page : int, optional - Default None. - Can be used to override the default behavior of all results being returned - to obtain a specific page of results. - version : int, optional - Version number for catalogs that have versions. Default is highest version. - resolver : str, optional - The resolver to use when resolving a named target into coordinates. Valid options are "SIMBAD" and "NED". - If not specified, the default resolver order will be used. Please see the - `STScI Archive Name Translation Application (SANTA) `__ - for more information. Default is None. + Deprecated. The page argument is no longer used. Please use ``offset`` instead. **criteria - Catalog-specific keyword args. - These can be found in the `service documentation `__. - for specific catalogs. For example, one can specify the magtype for an HSC search. - For catalogs available through Catalogs.MAST (PanSTARRS), the Column Name is the keyword, and the argument - should be either an acceptable value for that parameter, or a list consisting values, or tuples of - decorator, value pairs (decorator, value). In addition, columns may be used to select the return columns, - consisting of a list of column names. Results may also be sorted through the query with the parameter - sort_by composed of either a single Column Name to sort ASC, or a list of Column Nmaes to sort ASC or - tuples of Column Name and Direction (ASC, DESC) to indicate sort order (Column Name, DESC). - Detailed information of Catalogs.MAST criteria usage can - be found `here `__. + Keyword arguments representing criteria filters to apply. + + Criteria syntax + ---------------- + - Strings support wildcards using '*' (converted to SQL '%') and '%'. + - Lists are combined with OR for positive values; empty lists yield no matches. + - Numeric columns support comparison operators ('<', '<=', '>', '>=') and inclusive ranges using + the syntax 'low..high' (e.g., '5..10'). Mixed lists of numbers and comparisons are OR-combined. + - Negation: Prefix any value with '!' to negate that predicate. For list inputs, all negated values + for the same column are AND-combined, then ANDed with the OR of the positive values: + (neg1 AND neg2 AND ...) AND (pos1 OR pos2 OR ...). + + Examples + -------- + - file_suffix=['A', 'B', '!C'] -> (file_suffix != 'C') AND (file_suffix IN ('A', 'B')) + - size=['!14400', '<20000'] -> (size != 14400) AND (size < 20000) Returns ------- - response : list of `~requests.Response` + response : `~astropy.table.Table` + A table containing the query results. """ - - coordinates = utils.resolve_object(object_name, resolver=resolver) - - return self.query_region_async(coordinates, - radius=radius, - catalog=catalog, - version=version, - pagesize=pagesize, - page=page, - **criteria) + # Must specify one of region or coordinates + if region is None and coordinates is None: + raise InvalidQueryError( + "Must specify either `region` or `coordinates`. For non-positional queries, " + "use `Catalogs.query_criteria`." + ) + + # Parse pagination params + limit, offset = self._parse_legacy_pagination(limit, offset, pagesize, page) + + return self.query_criteria( + collection=collection, + catalog=catalog, + coordinates=coordinates, + region=region, + radius=radius, + limit=limit, + offset=offset, + count_only=count_only, + select_cols=select_cols, + sort_by=sort_by, + sort_desc=sort_desc, + filters=filters, + **criteria, + ) @class_or_instance - def query_criteria_async(self, catalog, *, pagesize=None, page=None, resolver=None, **criteria): + @deprecated_renamed_argument( + "version", + None, + since="0.4.12", + message="The `version` argument is deprecated and " + "will be removed in a future release. Please use `collection` and `catalog` instead.", + ) + @deprecated_renamed_argument( + "pagesize", + None, + since="0.4.12", + message="The `pagesize` argument is deprecated " + "and will be removed in a future release. Please use `limit` instead.", + ) + @deprecated_renamed_argument( + "page", + None, + since="0.4.12", + message="The `page` argument is deprecated " + "and will be removed in a future release. Please use `offset` instead.", + ) + @deprecated_renamed_argument("objectname", "object_name", since="0.4.12") + def query_object( + self, + object_name, + *, + radius=0.2 * u.deg, + collection=None, + catalog=None, + resolver=None, + limit=5000, + offset=0, + count_only=False, + select_cols=None, + sort_by=None, + sort_desc=False, + filters=None, + version=None, + pagesize=None, + page=None, + **criteria, + ): """ - Given an set of filters, returns a list of catalog entries. - See column documentation for specific catalogs `here `__. + Query for MAST catalog entries around a specified object name using criteria filters. To return columns + for a given collection and catalog, use `~astroquery.mast.CatalogsClass.get_column_metadata`. Parameters ---------- - catalog : str - The catalog to be queried. + object_name : str, optional + The name of the object to resolve and search around. + radius : str or `~astropy.units.Quantity` object, optional + The search radius around the target coordinates or object. Default 0.2 degrees. + collection : str, optional + The collection to be queried. If None, uses the instance's `collection` attribute. + catalog : str, optional + The catalog within the collection to query. If None, uses the instance's `catalog` attribute. + resolver : str, optional + The name resolver service to use when resolving ``object_name``. + limit : int, optional + The maximum number of results to return. Default is 5000. + offset : int, optional + The number of rows to skip before starting to return rows. Default is 0. + count_only : bool, optional + If True, only return the count of matching records instead of the records themselves. Default is False. + select_cols : list of str, optional + List of column names to include in the result. If None or empty, all columns are returned. + sort_by : str or list of str, optional + Column name(s) to sort the results by. + sort_desc : bool or list of bool, optional + Indicates whether to sort in descending order for each column in ``sort_by``. If a single bool, + applies to all columns. If a list, must match length of ``sort_by``. Default is False (ascending order). + filters : dict, optional + Another parameter to specify criteria filters as a dictionary. Use this option when the name of a column + conflicts with a named parameter of this method. + version : str, optional + Deprecated. The version argument is no longer used. Please use ``collection`` and ``catalog`` instead. pagesize : int, optional - Can be used to override the default pagesize. - E.g. when using a slow internet connection. + Deprecated. The pagesize argument is no longer used. Please use ``limit`` instead. page : int, optional - Can be used to override the default behavior of all results being returned to obtain - one specific page of results. - resolver : str, optional - The resolver to use when resolving a named target into coordinates. Valid options are "SIMBAD" and "NED". - If not specified, the default resolver order will be used. Please see the - `STScI Archive Name Translation Application (SANTA) `__ - for more information. Default is None. + Deprecated. The page argument is no longer used. Please use ``offset`` instead. **criteria - Criteria to apply. At least one non-positional criteria must be supplied. - Valid criteria are coordinates, object_name, radius (as in `query_region` and `query_object`), - and all fields listed in the column documentation for the catalog being queried. - The Column Name is the keyword, with the argument being one or more acceptable values for that parameter, - except for fields with a float datatype where the argument should be in the form [minVal, maxVal]. - For non-float type criteria wildcards maybe used (both * and % are considered wildcards), however - only one wildcarded value can be processed per criterion. - RA and Dec must be given in decimal degrees, and datetimes in MJD. - For example: filters=["FUV","NUV"],proposal_pi="Ost*",t_max=[52264.4586,54452.8914] - For catalogs available through Catalogs.MAST (PanSTARRS), the Column Name is the keyword, and the argument - should be either an acceptable value for that parameter, or a list consisting values, or tuples of - decorator, value pairs (decorator, value). In addition, columns may be used to select the return columns, - consisting of a list of column names. Results may also be sorted through the query with the parameter - sort_by composed of either a single Column Name to sort ASC, or a list of Column Nmaes to sort ASC or - tuples of Column Name and Direction (ASC, DESC) to indicate sort order (Column Name, DESC). - Detailed information of Catalogs.MAST criteria usage can - be found `here `__. + Keyword arguments representing criteria filters to apply. + + Criteria syntax + ---------------- + - Strings support wildcards using '*' (converted to SQL '%') and '%'. + - Lists are combined with OR for positive values; empty lists yield no matches. + - Numeric columns support comparison operators ('<', '<=', '>', '>=') and inclusive ranges using + the syntax 'low..high' (e.g., '5..10'). Mixed lists of numbers and comparisons are OR-combined. + - Negation: Prefix any value with '!' to negate that predicate. For list inputs, all negated values + for the same column are AND-combined, then ANDed with the OR of the positive values: + (neg1 AND neg2 AND ...) AND (pos1 OR pos2 OR ...). + + Examples + -------- + - file_suffix=['A', 'B', '!C'] -> (file_suffix != 'C') AND (file_suffix IN ('A', 'B')) + - size=['!14400', '<20000'] -> (size != 14400) AND (size < 20000) Returns ------- - response : list of `~requests.Response` + response : `~astropy.table.Table` + A table containing the query results. """ - - # Separating any position info from the rest of the filters - coordinates = criteria.pop('coordinates', None) - object_name = criteria.pop('object_name', None) - radius = criteria.pop('radius', 0.2*u.deg) - - if object_name or coordinates: - coordinates = utils.parse_input_location(coordinates=coordinates, - object_name=object_name, - resolver=resolver) - - # if radius is just a number we assume degrees - radius = coord.Angle(radius, u.deg) - - # build query - params = {} - if coordinates: - params["ra"] = coordinates.ra.deg - params["dec"] = coordinates.dec.deg - params["radius"] = radius.deg - - # Determine API connection, service name, and build filter set - filters = None - if catalog.lower() in self._service_api_connection.SERVICES: - self._current_connection = self._service_api_connection - service = catalog - - # validate user criteria - self._validate_service_criteria(catalog.lower(), **criteria) - - if not self._current_connection.check_catalogs_criteria_params(criteria): - raise InvalidQueryError("At least one non-positional criterion must be supplied.") - - for prop, value in criteria.items(): - params[prop] = value - - else: - self._current_connection = self._portal_api_connection - - if catalog.lower() == "tic": - service = "Mast.Catalogs.Filtered.Tic" - if coordinates or object_name: - service += ".Position" - service += ".Rows" # Using the rowstore version of the query for speed - column_config_name = "Mast.Catalogs.Tess.Cone" - params["columns"] = "*" - elif catalog.lower() == "ctl": - service = "Mast.Catalogs.Filtered.Ctl" - if coordinates or object_name: - service += ".Position" - service += ".Rows" # Using the rowstore version of the query for speed - column_config_name = "Mast.Catalogs.Tess.Cone" - params["columns"] = "*" - elif catalog.lower() == "diskdetective": - service = "Mast.Catalogs.Filtered.DiskDetective" - if coordinates or object_name: - service += ".Position" - column_config_name = "Mast.Catalogs.Dd.Cone" - else: - raise InvalidQueryError("Criteria query not available for {}".format(catalog)) - - filters = self._current_connection.build_filter_set(column_config_name, service, **criteria) - - if not filters: - raise InvalidQueryError("At least one non-positional criterion must be supplied.") - params["filters"] = filters - - # Parameters will be passed as JSON objects only when accessing the PANSTARRS API - use_json = catalog.lower() == 'panstarrs' - - return self._current_connection.service_request_async(service, params, pagesize=pagesize, page=page, - use_json=use_json) + # Parse pagination params + limit, offset = self._parse_legacy_pagination(limit, offset, pagesize, page) + + return self.query_criteria( + collection=collection, + catalog=catalog, + object_name=object_name, + radius=radius, + resolver=resolver, + limit=limit, + offset=offset, + count_only=count_only, + select_cols=select_cols, + sort_by=sort_by, + sort_desc=sort_desc, + filters=filters, + **criteria, + ) @class_or_instance + @deprecated(since="v0.4.12", message=("This function is deprecated and will be removed in a future release.")) def query_hsc_matchid_async(self, match, *, version=3, pagesize=None, page=None): """ Returns all the matches for a given Hubble Source Catalog MatchID. @@ -499,11 +705,10 @@ def query_hsc_matchid_async(self, match, *, version=3, pagesize=None, page=None) ------- response : list of `~requests.Response` """ - self._current_connection = self._portal_api_connection if isinstance(match, Row): - match = match["MatchID"] + match = match["MatchID"] if "MatchID" in match.colnames else match["matchid"] match = str(match) # np.int64 gives json serializer problems, so stringify right here if version == 2: @@ -515,9 +720,10 @@ def query_hsc_matchid_async(self, match, *, version=3, pagesize=None, page=None) params = {"input": match} - return self._current_connection.service_request_async(service, params, pagesize=pagesize, page=page) + return self._current_connection.service_request_async(service, params, pagesize, page) @class_or_instance + @deprecated(since="v0.4.12", message=("This function is deprecated and will be removed in a future release.")) def get_hsc_spectra_async(self, *, pagesize=None, page=None): """ Returns all Hubble Source Catalog spectra. @@ -535,14 +741,10 @@ def get_hsc_spectra_async(self, *, pagesize=None, page=None): ------- response : list of `~requests.Response` """ - self._current_connection = self._portal_api_connection + return self._current_connection.service_request_async("Mast.HscSpectra.Db.All", {}, pagesize, page) - service = "Mast.HscSpectra.Db.All" - params = {} - - return self._current_connection.service_request_async(service, params, pagesize, page) - + @deprecated(since="v0.4.12", message=("This function is deprecated and will be removed in a future release.")) def download_hsc_spectra(self, spectra, *, download_dir=None, cache=True, curl_flag=False): """ Download one or more Hubble Source Catalog spectra. @@ -566,108 +768,827 @@ def download_hsc_spectra(self, spectra, *, download_dir=None, cache=True, curl_f ------- response : list of `~requests.Response` """ - - # if spectra is not a Table, put it in a list + # Normalize spectra input to a list if isinstance(spectra, Row): spectra = [spectra] - # set up the download directory and paths - if not download_dir: - download_dir = '.' + # Ensure download directory is set + download_dir = download_dir or "." - if curl_flag: # don't want to download the files now, just the curl script + if curl_flag: + timestamp = time.strftime("%Y%m%d%H%M%S") + bundle_name = "mastDownload_" + timestamp + url_list = [self._make_data_url(spec) for spec in spectra] + path_list = [f"{bundle_name}/HSC/{spec['DatasetName']}.fits" for spec in spectra] - download_file = "mastDownload_" + time.strftime("%Y%m%d%H%M%S") + params = dict( + urlList=",".join(url_list), + filename=bundle_name, + pathList=",".join(path_list), + descriptionList=[""] * len(spectra), + productTypeList=["spectrum"] * len(spectra), + extension="curl", + ) - url_list = [] - path_list = [] - for spec in spectra: - if spec['SpectrumType'] < 2: - url_list.append('https://hla.stsci.edu/cgi-bin/getdata.cgi?config=ops&dataset={0}' - .format(spec['DatasetName'])) + service = "Mast.Bundle.Request" + response = self._portal_api_connection.service_request_async(service, params) + bundle_info = response[0].json() + local_script = os.path.join(download_dir, f"{bundle_name}.sh") + self._download_file(bundle_info["url"], local_script, head_safe=True) + + # Build manifest row + exists = os.path.isfile(local_script) + missing = [k for k, v in bundle_info.get("statusList", {}).items() if v != "COMPLETE"] + manifest = Table( + { + "Local Path": [local_script], + "Status": ["COMPLETE" if exists else "ERROR"], + "Message": [ + None + if exists and not missing + else ( + f"{len(missing)} files could not be added to curl script" + if exists + else "Curl script could not be downloaded" + ) + ], + "URL": [None if exists and not missing else (",".join(missing) if missing else bundle_info["url"])], + } + ) + return manifest + + base_dir = os.path.join(download_dir, "mastDownload", "HSC") + os.makedirs(base_dir, exist_ok=True) + manifest_rows = [] + + for row in spectra: + dataset = row["DatasetName"] + url = self._make_data_url(row) + local_path = os.path.join(base_dir, f"{dataset}.fits") + status = "COMPLETE" + message = None - else: - url_list.append('https://hla.stsci.edu/cgi-bin/ecfproxy?file_id={0}' - .format(spec['DatasetName']) + '.fits') + try: + self._download_file(url, local_path, cache=cache, head_safe=True) - path_list.append(download_file + "/HSC/" + spec['DatasetName'] + '.fits') + if not os.path.exists(local_path): + status = "ERROR" + message = "File was not downloaded" + except requests.HTTPError as err: + status = "ERROR" + message = f"HTTPError: {err}" - description_list = [""]*len(spectra) - producttype_list = ['spectrum']*len(spectra) + manifest_rows.append([local_path, status, message, url]) - service = "Mast.Bundle.Request" - params = {"urlList": ",".join(url_list), - "filename": download_file, - "pathList": ",".join(path_list), - "descriptionList": list(description_list), - "productTypeList": list(producttype_list), - "extension": 'curl'} + return Table(rows=manifest_rows, names=("Local Path", "Status", "Message", "URL")) - response = self._portal_api_connection.service_request_async(service, params) - bundler_response = response[0].json() + def _parse_result(self, response, *, verbose=False): + """Parse the async responses from HSC queries.""" + return self._current_connection._parse_result(response, verbose=verbose) - local_path = os.path.join(download_dir, "{}.sh".format(download_file)) - self._download_file(bundler_response['url'], local_path, head_safe=True) + def _verify_collection(self, collection): + """ + Verify that the specified collection is valid and return the correct collection name. + Warns the user if the collection has been renamed and raises an error if the collection is not valid. - status = "COMPLETE" - msg = None - url = None + Parameters + ---------- + collection : str + The collection to be verified. - if not os.path.isfile(local_path): - status = "ERROR" - msg = "Curl could not be downloaded" - url = bundler_response['url'] + Raises + ------ + InvalidQueryError + If the specified collection is not valid. + """ + collection = collection.lower().strip() + if collection in self.available_collections: + return collection + else: + if collection in self._renamed_collections: + new_name = self._renamed_collections[collection] + warn_msg = f"Collection '{collection}' has been renamed. Please use '{new_name}' instead." + warnings.warn(warn_msg, InputWarning) + return new_name + + error_msg = "" + if collection in self._no_longer_supported_collections: + error_msg = ( + f"Collection '{collection}' is no longer supported. To query from this catalog, " + f"please use a version of Astroquery older than 0.4.12." + ) else: - missing_files = [x for x in bundler_response['statusList'].keys() - if bundler_response['statusList'][x] != 'COMPLETE'] - if len(missing_files): - msg = "{} files could not be added to the curl script".format(len(missing_files)) - url = ",".join(missing_files) + closest = difflib.get_close_matches(collection, self.available_collections, n=1) + suggestion = f" Did you mean '{closest[0]}'?" if closest else "" + error_msg = f"Collection '{collection}' is not recognized.{suggestion}" + error_msg += " Available collections are: " + ", ".join(self.available_collections) + raise InvalidQueryError(error_msg) - manifest = Table({'Local Path': [local_path], - 'Status': [status], - 'Message': [msg], - "URL": [url]}) + def _get_collection_obj(self, collection_name): + """ + Given a collection name, find or create the corresponding CatalogCollection object. + Parameters + ---------- + collection_name : str + The name of the collection. + + Returns + ------- + CatalogCollection + The corresponding CatalogCollection object. + """ + if not isinstance(collection_name, str): + raise InvalidQueryError("Collection name must be a string.") + + collection_name = collection_name.lower().strip() + if collection_name in self._collections_cache: + return self._collections_cache[collection_name] + + collection_name = self._verify_collection(collection_name) + collection_obj = CatalogCollection(collection_name) + self._collections_cache[collection_name] = collection_obj + return collection_obj + + def _parse_inputs(self, collection=None, catalog=None): + """ + Parse and validate the collection and catalog inputs. + + Parameters + ---------- + collection : str, optional + The collection to be queried. If None, uses the instance's default collection. + catalog : str, optional + The catalog within the collection to query. If None, uses the instance's default catalog. + + Returns + ------- + tuple + A tuple containing the (collection, catalog) to be queried. + """ + collection_obj = self._get_collection_obj(collection) if collection else self._collection + + if not catalog: + # If the class attribute catalog is valid for the collection, use it + # Otherwise, use the default catalog for the collection + if self.catalog in collection_obj.catalog_names: + catalog = self.catalog + else: + catalog = collection_obj.default_catalog else: - base_dir = download_dir.rstrip('/') + "/mastDownload/HSC" + catalog = catalog.lower() + # For backwards compatibility, check if the user is trying to specify a collection via catalog + if ( + catalog in self.available_collections + or catalog in self._no_longer_supported_collections + or catalog in self._renamed_collections + ) and not collection: + warnings.warn( + f"Specifying collection '{catalog}' via the `catalog` parameter is deprecated. " + f"Please use the `collection` parameter instead.", + DeprecationWarning, + ) + # As a convenience to the user, set the collection accordingly and use its default catalog + collection_obj = self._get_collection_obj(catalog) + catalog = collection_obj.default_catalog + else: + catalog = collection_obj._verify_catalog(catalog) - if not os.path.exists(base_dir): - os.makedirs(base_dir) + return collection_obj, catalog - manifest_array = [] - for spec in spectra: + def _parse_select_cols(self, select_cols, column_metadata): + """ + Validate and parse the select_cols parameter. - if spec['SpectrumType'] < 2: - data_url = f'https://hla.stsci.edu/cgi-bin/getdata.cgi?config=ops&dataset={spec["DatasetName"]}' - else: - data_url = f'https://hla.stsci.edu/cgi-bin/ecfproxy?file_id={spec["DatasetName"]}.fits' + Parameters + ---------- + select_cols : list of str + List of column names to include in the result. + column_metadata : `~astropy.table.Table` + The catalog's column metadata table. + + Returns + ------- + str + Comma-separated string of valid column names for ADQL SELECT clause. + + Raises + ------ + InvalidQueryError + If any specified column is not found in the catalog metadata. + """ + valid_columns = column_metadata["column_name"].tolist() + valid_selected = [] + for col in select_cols: + if col not in valid_columns: + closest = difflib.get_close_matches(col, valid_columns, n=1) + suggestion = f" Did you mean '{closest[0]}'?" if closest else "" + warnings.warn(f"Column '{col}' not found in catalog.{suggestion}", InputWarning) + else: + valid_selected.append(col) + if not valid_selected: + raise InvalidQueryError("No valid columns specified in `select_cols`.") + return ", ".join(valid_selected) + + def _parse_legacy_pagination(self, limit, offset, pagesize, page): + """ + Parse legacy pagesize and page parameters to determine limit and offset. + + Parameters + ---------- + limit : int + The maximum number of results to return. + offset : int + The number of rows to skip before starting to return rows. + pagesize : int, optional + The number of results per page (legacy parameter). + page : int, optional + The page number to return (legacy parameter). - local_path = os.path.join(base_dir, f'{spec["DatasetName"]}.fits') + Returns + ------- + tuple + A tuple containing the (limit, offset) values. + """ + # If limit and offset are default, check for legacy pagination params + if limit == 5000 and offset == 0: + if pagesize is not None: + if page is None: + page = 1 # Default to first page if not specified + limit = pagesize + offset = (page - 1) * pagesize + elif page is not None: + warnings.warn( + "The 'page' parameter is ignored without 'pagesize'. " + "Please use `limit` and `offset` to specify pagination.", + InputWarning, + ) + return limit, offset + + def _create_adql_region(self, region): + """ + Returns the ADQL description of the given polygon or circle region. + + Parameters + ---------- + region : str | iterable | astropy.regions.Region + - Iterable of RA/Dec pairs as lists/sequences + - STC-S POLYGON or CIRCLE string + - `~astropy.regions.CircleSkyRegion` or `~astropy.regions.PolygonSkyRegion` + + Returns + ------- + adql_region : str + ADQL representation of the region (POLYGON or CIRCLE) + """ + # Case 1: region is a string (e.g. STC-S syntax) + if isinstance(region, str): + parts = region.strip().lower().split() + shape = parts[0] + + if shape == "polygon": + # POLYGON lon1 lat1 lon2 lat2 ... + # Optional format: POLYGON ICRS lon1 lat1 ... + # parts = ["POLYGON", maybe_frame?, ...coords...] + + # Determine if parts[1] is a coord or a frame name + if len(parts) < 3: + raise InvalidQueryError(f"Invalid POLYGON region string: {region}") + try: + float(parts[1]) # numeric → no frame name + point_parts = parts[1:] + except ValueError: + point_parts = parts[2:] # skip optional frame name + + if len(point_parts) < 6 or len(point_parts) % 2 != 0: + # polygon requires at least 3 points (6 numbers), and must be pairs + raise InvalidQueryError(f"Invalid POLYGON region string: {region}") - status = "COMPLETE" - msg = None - url = None + point_string = ",".join(point_parts) + return f"POLYGON('ICRS',{point_string})" + elif shape == "circle": + # CIRCLE ra dec radius (or CIRCLE ICRS ra dec radius) + if len(parts) < 4: + raise InvalidQueryError(f"Invalid CIRCLE region string: {region}") + + # Try interpreting parts[1] as RA. If not numeric, assume it's a frame try: - self._download_file(data_url, local_path, cache=cache, head_safe=True) + float(parts[1]) + # Format: CIRCLE ra dec radius + ra, dec, radius = parts[1], parts[2], parts[3] + except ValueError: + # Format: CIRCLE FRAME ra dec radius + if len(parts) < 5: + raise InvalidQueryError(f"Invalid CIRCLE region string: {region}") + ra, dec, radius = parts[2], parts[3], parts[4] - # check file size also this is where would perform md5 - if not os.path.isfile(local_path): - status = "ERROR" - msg = "File was not downloaded" - url = data_url + return f"CIRCLE('ICRS',{ra},{dec},{radius})" - except HTTPError as err: - status = "ERROR" - msg = "HTTPError: {0}".format(err) - url = data_url + else: + raise InvalidQueryError(f"Unrecognized region string: {region}") + + # Case 2: region is an astropy region object + # TODO: When released, change these to use `CircleSphericalSkyRegion` and `PolygonSphericalSkyRegion` + if HAS_REGIONS: + if isinstance(region, CircleSkyRegion): + center = region.center.icrs + radius = region.radius.to(u.deg).value + return f"CIRCLE('ICRS',{center.ra.deg},{center.dec.deg},{radius})" + elif isinstance(region, PolygonSkyRegion): + verts = region.vertices.icrs + point_string = ",".join(f"{v.ra.deg},{v.dec.deg}" for v in verts) + return f"POLYGON('ICRS',{point_string})" + + # Case 3: region is an iterable of coordinate pairs + if isinstance(region, Iterable): + # Expect something like [(ra1, dec1), (ra2, dec2), ...] + try: + points = [float(x) for point in region for x in point] + except Exception as e: + raise InvalidQueryError(f"Invalid iterable region format: {region}") from e + return f"POLYGON('ICRS',{','.join(str(x) for x in points)})" + + else: + raise TypeError(f"Unsupported region type: {type(region)}") + + def _classify_columns(self, meta): + """ + Classify columns as numeric or temporal based on their datatype and UCD metadata. + + Parameters + ---------- + meta : `~astropy.table.Table` + The catalog's column metadata table, which must include 'column_name', 'datatype', and 'ucd' columns. + + Returns + ------- + tuple + A tuple containing two sets: (numeric_columns, temporal_columns), where each set contains the names of the + columns classified as numeric or temporal, respectively. + """ + num_types = {"int", "long", "short", "float", "double", "floatcomplex", "doublecomplex"} + temporal_tokens = {"date", "time", "timestamp", "datetime"} + + numeric = set() + temporal = set() + + for name, dtype, ucd in zip(meta["column_name"], meta["datatype"], meta["ucd"]): + dtype_str = dtype.lower() if isinstance(dtype, str) else "" + ucd_str = ucd.lower() if isinstance(ucd, str) else "" + + # Classify as numeric if the datatype matches known numeric types + if dtype_str in num_types: + numeric.add(name) + + # Classify as temporal if the datatype contains temporal tokens or if the + # UCD indicates a time-related quantity (but exclude numeric types that may have time-related UCDs) + has_temporal_dtype = any(token in dtype_str for token in temporal_tokens) + has_temporal_ucd = ucd_str.startswith("time.") or ";time." in ucd_str + not_numeric = dtype_str not in num_types + + if has_temporal_dtype or (has_temporal_ucd and not_numeric): + temporal.add(name) + + return numeric, temporal + + def _quote_adql_string(self, adql_str): + """Escape single quotes in ADQL query strings by doubling them.""" + return adql_str.replace("'", "''") + + def _parse_range_cmp_expr(self, col, expr, value_formatter): + expr = expr.strip() + + range_match = re.fullmatch(r"(.+?)\s*\.\.\s*(.+)", expr) + if range_match: + low = value_formatter(range_match.group(1).strip(), col) + high = value_formatter(range_match.group(2).strip(), col) + return f"{col} BETWEEN {low} AND {high}" + + cmp_match = re.fullmatch(r"(<=|>=|<|>)\s*(.+)", expr) + if cmp_match: + rhs = value_formatter(cmp_match.group(2).strip(), col) + return f"{col} {cmp_match.group(1)} {rhs}" + + def _parse_numeric_expr(self, col, expr): + """ + Parse a numeric expression for a column and return the corresponding ADQL predicate. + + Parameters + ---------- + col : str + The column name. + expr : str + The numeric expression (e.g., "5", "<10", "5..10"). + + Returns + ------- + str + The ADQL predicate for the numeric expression. + """ + parsed = self._parse_range_cmp_expr(col, expr, lambda x, _: x) + if parsed: + return parsed + + try: + return f"{col} = {float(expr)}" + except ValueError: + raise InvalidQueryError( + f"Column '{col}' is numeric; unsupported value '{expr}'. Use numbers, comparisons like '<10', or " + "ranges like '5..10'." + ) + + def _normalize_time(self, value): + """ + Normalize a datetime value to a string format suitable for ADQL queries. + + Parameters + ---------- + value : str or `~astropy.time.Time` or `~datetime.datetime` + The datetime value to normalize. + + Returns + ------- + str + The normalized datetime string in the format 'YYYY-MM-DD HH:MM:SS' or the original + value if it cannot be parsed as a datetime. + + Raises + ------ + ValueError + If the value cannot be parsed as a datetime. + """ + t = Time(value) + dt = t.to_datetime() - manifest_array.append([local_path, status, msg, url]) + # Drop microseconds to avoid ADQL parsing issues + if dt.microsecond: + dt = dt.replace(microsecond=0) - manifest = Table(rows=manifest_array, names=('Local Path', 'Status', 'Message', "URL")) + return dt.strftime("%Y-%m-%d %H:%M:%S") - return manifest + def _to_temporal_literal(self, value, col): + """ + Convert a datetime value to an ADQL literal string, ensuring it is properly formatted and quoted. + + Parameters + ---------- + value : str or `~astropy.time.Time` or `~datetime.datetime` + The datetime value to convert. + col : str + The column name (used for error messages). + + Returns + ------- + str + The ADQL literal string representing the datetime value. + """ + try: + normalized = self._normalize_time(value) + except ValueError: + # If it can't be parsed as a time, send the value as-is for the backend to handle + # (which may raise an error if it's invalid) + normalized = value if isinstance(value, str) else str(value) + escaped = self._quote_adql_string(normalized) + return f"'{escaped}'" + + def _parse_temporal_expr(self, col, expr): + """ + Parse a datetime expression for a column and return the corresponding ADQL predicate. + + Parameters + ---------- + col : str + The column name. + expr : str + Datetime expression (e.g., "2024-01-01", ">=2024-01-01", "2024-01-01..2024-12-31"). + + Returns + ------- + str + The ADQL predicate for the datetime expression. + """ + if isinstance(expr, str): + parsed = self._parse_range_cmp_expr(col, expr, self._to_temporal_literal) + if parsed: + return parsed + + # For simple equality, expand to a small range + try: + t0 = Time(self._normalize_time(expr)) + t1 = t0 + 1 * u.s + low = self._quote_adql_string(t0.strftime("%Y-%m-%d %H:%M:%S")) + high = self._quote_adql_string(t1.strftime("%Y-%m-%d %H:%M:%S")) + return f"{col} BETWEEN '{low}' AND '{high}'" + except ValueError: + # If it can't be parsed as a time, send the value as-is for the backend to handle + # (which may raise an error if it's invalid) + return f"{col} = '{expr}'" + + def _format_scalar_predicate(self, col, val, is_numeric=False, is_temporal=False): + """ + Build predicate for a scalar value, aware of column type. + + Parameters + ---------- + col : str + The column name. + val : scalar + The value to build the predicate for. + is_numeric : bool + Whether the column is numeric. + is_temporal : bool + Whether the column is temporal. + + Returns + ------- + str + The ADQL predicate for the scalar value. + """ + if isinstance(val, bool): + # Booleans stored as integers + return f"{col} = {int(val)}" + + if is_temporal: + is_neg = isinstance(val, str) and val.startswith("!") + sval = val[1:].strip() if is_neg else val + parsed = self._parse_temporal_expr(col, sval) + return f"NOT ({parsed})" if is_neg else parsed + + if isinstance(val, str): + # Check for negation + is_neg = val.startswith("!") + sval = val[1:].strip() if is_neg else val + + # Strings for numeric columns + if is_numeric: + parsed = self._parse_numeric_expr(col, sval) + return f"NOT ({parsed})" if is_neg else parsed + + # Non-numeric strings + has_wild = ("*" in sval) or ("%" in sval) + pattern = self._quote_adql_string(sval.replace("*", "%")) + expr = f"{col} LIKE '{pattern}'" if has_wild else f"{col} = '{pattern}'" + return f"NOT ({expr})" if is_neg else expr + + # Numerics or others + return f"{col} = {val}" + + def _combine_predicates(self, pos_parts, neg_parts): + """ + Combine positive and negative predicate parts into a single ADQL expression. + + Parameters + ---------- + pos_parts : list of str + List of positive predicate strings. + neg_parts : list of str + List of negative predicate strings. + + Returns + ------- + str + The combined ADQL predicate. + """ + pos_expr = "" + if len(pos_parts) == 1: + pos_expr = pos_parts[0] + elif len(pos_parts) > 1: + pos_expr = "(" + " OR ".join(pos_parts) + ")" + + if neg_parts and pos_expr: + return "(" + " AND ".join(neg_parts) + ") AND " + pos_expr + if neg_parts: + return " AND ".join(neg_parts) + return pos_expr + + def _build_list_predicate(self, col, pos_items, neg_items, pos_builder, neg_builder): + """ + General helper to build predicates for list values with separate positive and negative handling. + + Parameters + ---------- + col : str + The column name. + pos_items : list + List of positive values. + neg_items : list + List of negative values. + pos_builder : function + Function to build positive predicates, called as pos_builder(col, pos_items). + neg_builder : function + Function to build negative predicates, called as neg_builder(col, neg_items). + + Returns + ------- + str + The combined ADQL predicate for the list values. + """ + pos_parts = pos_builder(col, pos_items) if pos_items else [] + neg_parts = [neg_builder(col, v) for v in neg_items] if neg_items else [] + return self._combine_predicates(pos_parts, neg_parts) + + def _build_numeric_list_predicate(self, col, pos_items, neg_items): + """ + Build predicate for multiple values passed into a numeric column with separated positives and negatives. + + Parameters + ---------- + col : str + The column name. + pos_items : list + List of positive values. + neg_items : list + List of negative values. + + Returns + ------- + str + The ADQL predicate for the numeric list. + """ + def build_positive(col, items): + # Separate simple numeric values from complex expressions (comparisons, ranges) + simple_numbers = [] + complex_parts = [] + for val in items: + if isinstance(val, bool): + simple_numbers.append(int(val)) + elif isinstance(val, (int, float)): + simple_numbers.append(val) + elif isinstance(val, str): + try: + simple_numbers.append(float(val)) + except ValueError: + complex_parts.append(self._parse_numeric_expr(col, val)) + else: + try: + simple_numbers.append(float(val)) + except (ValueError, TypeError): + raise InvalidQueryError(f"Unsupported numeric value type: {type(val)}") + + parts = [] + if simple_numbers: + vals = [str(v) for v in simple_numbers] + parts.append(f"{col} IN (" + ", ".join(vals) + ")") + parts.extend(complex_parts) + return parts + + def build_negative(col, v): + # For negatives, we can't use NOT IN or NOT (complex), so we build each as a separate predicate + # and AND them together + return self._format_scalar_predicate(col, f"!{v}", is_numeric=True) + + return self._build_list_predicate( + col, + pos_items, + neg_items, + build_positive, + build_negative, + ) + + def _build_string_list_predicate(self, col, pos_items, neg_items): + """ + Build predicate for multiple values passed into a string column with separated positives and negatives. + + Parameters + ---------- + col : str + The column name. + pos_items : list + List of positive values. + neg_items : list + List of negative values. + + Returns + ------- + str + The ADQL predicate for the string list. + """ + def build_positive(col, items): + # Separate simple strings (no wildcards) from those that require LIKE + simple_strings = [] + pattern_parts = [] + for v in items: + if isinstance(v, bool): + simple_strings.append(str(int(v))) + elif isinstance(v, str): + if ("*" in v) or ("%" in v): + patt = self._quote_adql_string(v.replace("*", "%")) + pattern_parts.append(f"{col} LIKE '{patt}'") + else: + simple_strings.append("'" + self._quote_adql_string(v) + "'") + else: + simple_strings.append(str(v)) + + parts = [] + if simple_strings: + parts.append(f"{col} IN (" + ", ".join(simple_strings) + ")") + parts.extend(pattern_parts) + return parts + + # Negative predicates → use helper + def build_negative(col, v): + # For negatives, we can't use NOT IN or NOT (LIKE), so we build each as a separate predicate + # and AND them together + return self._format_scalar_predicate(col, f"!{v}") + + return self._build_list_predicate( + col, + pos_items, + neg_items, + build_positive, + build_negative, + ) + + def _build_temporal_list_predicate(self, col, pos_items, neg_items): + """ + Build temporal list predicates using column datatype to choose CAST vs string comparison. + + Parameters + ---------- + col : str + The column name. + pos_items : list + List of positive values. + neg_items : list + List of negative values. + + Returns + ------- + str + The ADQL predicate for the temporal list. + """ + return self._build_list_predicate( + col, + pos_items, + neg_items, + pos_builder=lambda c, vals: [self._parse_temporal_expr(c, v) for v in vals], + neg_builder=lambda c, v: self._format_scalar_predicate(c, f"!{v}", is_temporal=True), + ) + + def _format_criteria_conditions(self, collection_obj, catalog, criteria): + """ + Turn a criteria dict into ADQL WHERE clause expressions, aware of column types. + + - Scalars: equality (strings quoted; booleans -> 0/1; numerics raw). + - Strings with wildcards '*' or '%': uses LIKE (converting '*' to '%'). + - Lists/Tuples: if any string contains wildcard, build OR of LIKEs; otherwise use IN (...). + - Numeric columns: support comparison strings ('<10', '>= 5') and ranges ('5..10', inclusive). + Empty lists yield a false predicate (1=0). + - Negation: a value prefixed with '!' is treated as a negated predicate. For list values, all negations are + AND'ed together and combined with the OR of positives: (neg1 AND neg2) AND (pos1 OR pos2 ...). + + Parameters + ---------- + criteria : dict + Mapping of column name -> scalar or list of scalars. + + Returns + ------- + list of str + ADQL predicate strings (without leading WHERE/AND), suitable for joining with ' AND '. + """ + column_meta = collection_obj.get_catalog_metadata(catalog).column_metadata + numeric_cols, temporal_cols = self._classify_columns(column_meta) + conditions = [] + for key, value in criteria.items(): + # Handle list-like values => IN or OR(LIKE ...) + if isinstance(value, (list, tuple)): + values = list(value) + if len(values) == 0: + conditions.append("1=0") + continue + # Separate negatives (prefixed with '!') and positives + neg_items = [] + pos_items = [] + for v in values: + if isinstance(v, str) and v.startswith("!"): + neg_items.append(v[1:].strip()) + else: + pos_items.append(v) + + if key in temporal_cols: + expr = self._build_temporal_list_predicate(key, pos_items, neg_items) + if expr: + conditions.append(expr) + elif key in numeric_cols: + expr = self._build_numeric_list_predicate(key, pos_items, neg_items) + if expr: + conditions.append(expr) + else: + expr = self._build_string_list_predicate(key, pos_items, neg_items) + if expr: + conditions.append(expr) + else: + conditions.append(self._format_scalar_predicate(key, value, key in numeric_cols, key in temporal_cols)) + return conditions + + def _make_data_url(self, row): + """Return the correct data URL for a given spectrum row.""" + dataset = row["DatasetName"] + if row["SpectrumType"] < 2: + return f"https://hla.stsci.edu/cgi-bin/getdata.cgi?config=ops&dataset={dataset}" + return f"https://hla.stsci.edu/cgi-bin/ecfproxy?file_id={dataset}.fits" Catalogs = CatalogsClass() diff --git a/astroquery/mast/tests/data/README.rst b/astroquery/mast/tests/data/README.rst index 5e0a21f6d4..84b82bc727 100644 --- a/astroquery/mast/tests/data/README.rst +++ b/astroquery/mast/tests/data/README.rst @@ -62,3 +62,66 @@ To generate `~astroquery.mast.tests.data.resolver.json`, use the following: ... {'name': objects, 'outputFormat': 'json', 'resolveAll': 'true'}) >>> with open('resolver.json', 'w') as file: ... json.dump(resp.json(), file, indent=4) # doctest: +SKIP + +To generate `~astroquery.mast.tests.data.tap_collections.json`, use the following: + +.. doctest-remote-data:: + + >>> import json + >>> from astroquery.mast import utils + ... + >>> resp = utils._simple_request('https://mast.stsci.edu/vo-tap/api/v0.1/openapi.json') + ... + >>> with open('tap_collections.json', 'w') as file: + ... json.dump(resp.json(), file, indent=4) # doctest: +SKIP + +To generate `~astroquery.mast.tests.data.tap_catalogs.vot`, use the following: + +.. doctest-remote-data:: + + >>> import pyvo + >>> from astropy.io.votable import writeto + ... + >>> tap_service = pyvo.dal.TAPService("https://mast.stsci.edu/vo-tap/api/v0.1/tic") + >>> query = 'SELECT table_name, description FROM tap_schema.tables' + >>> result = tap_service.run_sync(query) + >>> writeto(result.votable, 'tap_catalogs.vot') # doctest: +SKIP + +To generate `~astroquery.mast.tests.data.tap_columns.vot`, use the following: + +.. doctest-remote-data:: + + >>> import pyvo + >>> from astropy.io.votable import writeto + ... + >>> tap_service = pyvo.dal.TAPService("https://mast.stsci.edu/vo-tap/api/v0.1/tic") + >>> query = "SELECT column_name, datatype, unit, ucd, description FROM tap_schema.columns WHERE table_name = 'dbo.catalogrecord'" + >>> result = tap_service.run_sync(query) + >>> writeto(result.votable, 'tap_columns.vot') # doctest: +SKIP + +To generate `~astroquery.mast.tests.data.tap_capabilities.xml`, use the following: + +.. doctest-remote-data:: + + >>> import requests + >>> import pyvo + ... + >>> tap_service = pyvo.dal.TAPService("https://mast.stsci.edu/vo-tap/api/v0.1/tic") + >>> caps_url = tap_service.baseurl.rstrip("/") + "/capabilities" + >>> resp = requests.get(caps_url) + >>> resp.raise_for_status() + ... + >>> with open("tap_capabilities.xml", "wb") as f: + ... f.write(resp.content) # doctest: +SKIP + +To generate `~astroquery.mast.tests.data.tap_results.vot`, use the following: + +.. doctest-remote-data:: + + >>> import pyvo + >>> from astropy.io.votable import writeto + ... + >>> tap_service = pyvo.dal.TAPService("https://mast.stsci.edu/vo-tap/api/v0.1/tic") + >>> query = "SELECT TOP 10 * FROM dbo.catalogrecord WHERE CONTAINS(POINT('ICRS', ra, dec), CIRCLE('ICRS', 23.34086, 60.658, 0.002)) = 1" + >>> result = tap_service.run_sync(query) + >>> writeto(result.votable, 'tap_results.vot') # doctest: +SKIP diff --git a/astroquery/mast/tests/data/tap_capabilities.xml b/astroquery/mast/tests/data/tap_capabilities.xml new file mode 100644 index 0000000000..60911ad49c --- /dev/null +++ b/astroquery/mast/tests/data/tap_capabilities.xml @@ -0,0 +1 @@ +https://masttest.stsci.edu/vo-tap/api/v0.1/ticADQL2.0ADQL-2.0
CONTAINS
POINT
CIRCLE
application/jsonjsontext/csv;header=presentcsvapplication/xmlxmlapplication/x-votable+xmlvotable100000100000https://masttest.stsci.edu/vo-tap/api/v0.1/tic/capabilitieshttps://masttest.stsci.edu/vo-tap/api/v0.1/tic/availabilityhttps://masttest.stsci.edu/vo-tap/api/v0.1/tic/tableshttps://masttest.stsci.edu/vo-tap/api/v0.1/tic/examples \ No newline at end of file diff --git a/astroquery/mast/tests/data/tap_catalogs.vot b/astroquery/mast/tests/data/tap_catalogs.vot new file mode 100644 index 0000000000..21dbeffe59 --- /dev/null +++ b/astroquery/mast/tests/data/tap_catalogs.vot @@ -0,0 +1,49 @@ + + + + + + + + + + Fully qualified table name + + + + + Brief description of the table + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
tap_schema.schemasdescription of schemas in this dataset
tap_schema.tablesdescription of tables in this dataset
tap_schema.columnsdescription of columns in this dataset
tap_schema.keysdescription of foreign keys in this dataset
tap_schema.key_columnsdescription of foreign key columns in this dataset
dbo.catalogrecordMain Catalog Record
+ + diff --git a/astroquery/mast/tests/data/tap_collections.json b/astroquery/mast/tests/data/tap_collections.json new file mode 100644 index 0000000000..73509c6a97 --- /dev/null +++ b/astroquery/mast/tests/data/tap_collections.json @@ -0,0 +1,1960 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "tap_search", + "version": "0.1" + }, + "paths": { + "/{catalog}/": { + "get": { + "tags": [ + "sync", + "sync" + ], + "summary": "Taphandler.Get Site Page", + "description": "Returns the service query manager doc page", + "operationId": "TAPHandler_get_site_page__catalog___get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view example queries for." + }, + "description": "Catalog to view example queries for." + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/sync": { + "get": { + "tags": [ + "sync", + "sync" + ], + "summary": "Taphandler.Get", + "description": "VO TAP Search", + "operationId": "TAPHandler_get__catalog__sync_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "catalog to search" + }, + "description": "catalog to search" + }, + { + "name": "lang", + "in": "query", + "required": true, + "schema": { + "$ref": "#/components/schemas/LangName", + "description": "Query language to be used for this request" + }, + "description": "Query language to be used for this request" + }, + { + "name": "responseformat", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/EncodingEnum", + "description": "Content type of the response", + "default": "votable" + }, + "description": "Content type of the response" + }, + { + "name": "format", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/EncodingEnum", + "description": "Content type of the response", + "default": "votable" + }, + "description": "Content type of the response" + }, + { + "name": "query", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "Query to perform", + "title": "Query" + }, + "description": "Query to perform" + }, + { + "name": "maxrec", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "minimum": 0, + "description": "Maximum number of records to be returned (max 100k)", + "default": 100000, + "title": "Maxrec" + }, + "description": "Maximum number of records to be returned (max 100k)" + }, + { + "name": "runid", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "maxLength": 64 + }, + { + "type": "null" + } + ], + "description": "RUNID for the request", + "title": "Runid" + }, + "description": "RUNID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "post": { + "tags": [ + "sync", + "sync" + ], + "summary": "Taphandler.Post", + "description": "VO TAP Search (FORM)", + "operationId": "TAPHandler_post__catalog__sync_post", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "catalog to search" + }, + "description": "catalog to search" + } + ], + "requestBody": { + "required": true, + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/Body_TAPHandler_post__catalog__sync_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response" + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/examples": { + "get": { + "tags": [ + "sync", + "sync" + ], + "summary": "Exampleshandler.Get", + "description": "Page displaying example TAP queries.\n\nThese example pages conform to the DALI-examples spec and are used to provide both human and machine-readable\nexample queries to the user.\n\nhttps://www.ivoa.net/documents/DALI/20170517/REC-DALI-1.1.html#tth_sEc2.3\n\nDuring automated testing, the queries on this page are stripped from the XHTML and used as test queries.\nThe XHTML element tags of the queries are specific to this testing/spec and should not be changed without\nconsulting the above specification, then also changing the necessary tests.", + "operationId": "ExamplesHandler_get__catalog__examples_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view example queries for." + }, + "description": "Catalog to view example queries for." + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/tables": { + "get": { + "tags": [ + "vosi", + "vosi" + ], + "summary": "Vositableshandler.Get All Tables", + "description": "Return information on all available tables.", + "operationId": "VOSITablesHandler_get_all_tables__catalog__tables_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view VOSI table info for." + }, + "description": "Catalog to view VOSI table info for." + }, + { + "name": "detail", + "in": "query", + "required": false, + "schema": { + "type": "string", + "description": "Amount of table detail to return.", + "default": "min", + "title": "Detail" + }, + "description": "Amount of table detail to return." + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/tables/{table}": { + "get": { + "tags": [ + "vosi", + "vosi" + ], + "summary": "Vositableshandler.Get Table", + "description": "Return information for a specific table.", + "operationId": "VOSITablesHandler_get_table__catalog__tables__table__get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view VOSI table info for." + }, + "description": "Catalog to view VOSI table info for." + }, + { + "name": "table", + "in": "path", + "required": true, + "schema": { + "type": "string", + "description": "The specific table to view info for.", + "title": "Table" + }, + "description": "The specific table to view info for." + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/availability": { + "get": { + "tags": [ + "vosi", + "vosi" + ], + "summary": "Vosiavailabilityhandler.Get Availability", + "description": "Availability metadata defined in the VOSI 1.1 spec at:\nhttps://www.ivoa.net/documents/VOSI/20170524/REC-VOSI-1.1.html#tth_sEc3.2", + "operationId": "VOSIAvailabilityHandler_get_availability__catalog__availability_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view VOSI availability info for." + }, + "description": "Catalog to view VOSI availability info for." + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/capabilities": { + "get": { + "tags": [ + "vosi", + "vosi" + ], + "summary": "Vosicapabilitieshandler.Get Capabilities", + "description": "TAP specific capability metadata as described in the service VOResource", + "operationId": "VOSICapabilitiesHandler_get_capabilities__catalog__capabilities_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "type": "string", + "description": "Catalog to view VOSI capability info for.", + "title": "Catalog" + }, + "description": "Catalog to view VOSI capability info for." + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Async", + "description": "Returns Jobs list per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_async__catalog__async_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view example queries for." + }, + "description": "Catalog to view example queries for." + }, + { + "name": "phase", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/components/schemas/ExecutionPhase" + } + }, + { + "type": "null" + } + ], + "description": "The current execution phase to filter jobs by.", + "title": "Phase" + }, + "description": "The current execution phase to filter jobs by." + }, + { + "name": "after", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ], + "description": "Return jobs with creation times after the given date.", + "title": "After" + }, + "description": "Return jobs with creation times after the given date." + }, + { + "name": "last", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer", + "exclusiveMinimum": 0 + }, + { + "type": "null" + } + ], + "description": "Return the given number of most recently created jobs.", + "title": "Last" + }, + "description": "Return the given number of most recently created jobs." + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "post": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Post", + "description": "VO TAP Search, asynchronous job setup via form", + "operationId": "TAPAsyncHandler_post__catalog__async_post", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to query." + }, + "description": "Catalog to query." + } + ], + "requestBody": { + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/Body_TAPAsyncHandler_post__catalog__async_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/results/result": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Results", + "description": "Returns job results if existent, per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_results__catalog__async__job_id__results_result_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/results": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Results", + "description": "Return job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info_detail_results__catalog__async__job_id__results_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/error": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Error", + "description": "Return job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding\nIn case of a valid job with no errors, an empty 200 OK response is returned.\n\nFor a TAP service, the response should match the requested format, if specified.\nhttps://www.ivoa.net/documents/TAP/20190927/REC-TAP-1.1.html#tth_sEc3.3", + "operationId": "TAPAsyncHandler_get_job_info_detail_error__catalog__async__job_id__error_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/phase": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Phase", + "description": "Return job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info_detail_phase__catalog__async__job_id__phase_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "post": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Post Update Job Phase", + "description": "Job control for existing jobs via POST per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_post_update_job_phase__catalog__async__job_id__phase_post", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog associated with this job." + }, + "description": "Catalog associated with this job." + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "requestBody": { + "required": true, + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/Body_TAPAsyncHandler_post_update_job_phase__catalog__async__job_id__phase_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/executionduration": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Executionduration", + "description": "Execution Duration is a potentially client-negotiated timeout value in seconds.\nIf it is not explicitly set by the client, a server default is used.\nReturn job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info_detail_executionduration__catalog__async__job_id__executionduration_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "post": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Post Update Job Execution Duration", + "description": "Updates job execution timeout, in seconds.\nOnly has an effect before the job is set to Phase RUN.\nNot supported on all catalogs. Limited by a server side max.\nJob control for existing jobs via POST per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_post_update_job_execution_duration__catalog__async__job_id__executionduration_post", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog associated with this job." + }, + "description": "Catalog associated with this job." + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "requestBody": { + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/Body_TAPAsyncHandler_post_update_job_execution_duration__catalog__async__job_id__executionduration_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/destruction": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Destruction", + "description": "Return job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info_detail_destruction__catalog__async__job_id__destruction_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "post": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Post Update Job Destruction", + "description": "Job control for existing jobs via POST per UWS spec:\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#DestructionTime\nThe service may forbid changes, or may set limits on the allowed destruction time.\nDestruction datetime is expected in ISO 8601 UTC format with Z for UTC times:\nhttps://www.ivoa.net/documents/VOResource/20180625/REC-VOResource-1.1.html#tth_sEc2.2.4", + "operationId": "TAPAsyncHandler_post_update_job_destruction__catalog__async__job_id__destruction_post", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view example queries for." + }, + "description": "Catalog to view example queries for." + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "requestBody": { + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/Body_TAPAsyncHandler_post_update_job_destruction__catalog__async__job_id__destruction_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/quote": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Quote", + "description": "Job completion time estimate quote not provided by this service, returns empty value.\nReturn job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info_detail_quote__catalog__async__job_id__quote_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/parameters": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Parameters", + "description": "Return job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info_detail_parameters__catalog__async__job_id__parameters_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "post": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Update Job Info Parameters", + "description": "Update job parameters by submitting a POST of key-value pairs.", + "operationId": "TAPAsyncHandler_update_job_info_parameters__catalog__async__job_id__parameters_post", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying" + }, + "description": "Catalog for querying" + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "requestBody": { + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/Body_TAPAsyncHandler_update_job_info_parameters__catalog__async__job_id__parameters_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}/owner": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info Detail Owner", + "description": "Return job details per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info_detail_owner__catalog__async__job_id__owner_get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog for querying." + }, + "description": "Catalog for querying." + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/{catalog}/async/{job_id}": { + "get": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Get Job Info", + "description": "Returns specific Job info per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_get_job_info__catalog__async__job_id__get", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view example queries for." + }, + "description": "Catalog to view example queries for." + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + }, + { + "name": "wait", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Maximum time in seconds to block until a job status change", + "title": "Wait" + }, + "description": "Maximum time in seconds to block until a job status change" + }, + { + "name": "phase", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/ExecutionPhase" + }, + { + "type": "null" + } + ], + "description": "For blocking behavior, can supply the current execution phase to monitor for changes", + "title": "Phase" + }, + "description": "For blocking behavior, can supply the current execution phase to monitor for changes" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "post": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Post Update Job Info", + "description": "Job control for existing jobs via POST per UWS spec\nhttps://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#RESTbinding", + "operationId": "TAPAsyncHandler_post_update_job_info__catalog__async__job_id__post", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view example queries for." + }, + "description": "Catalog to view example queries for." + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "requestBody": { + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/Body_TAPAsyncHandler_post_update_job_info__catalog__async__job_id__post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "async", + "async" + ], + "summary": "Tapasynchandler.Delete Job Route", + "description": "Deletes specific Job info per UWS spec. Cancels if running. Note the /jobs path for TAP is just /async\nPer https://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html#d1e1390:\nSending a HTTP DELETE to a Job resource destroys that job, with the meaning noted in the definition\nof the Job object, above. No other resource of the UWS may be directly deleted by the client.\nThe response to this request must have code 303 \u201cSee other\u201d and the Location header of the response\nmust point to the Job List at the /{jobs} URI.", + "operationId": "TAPAsyncHandler_delete_job_route__catalog__async__job_id__delete", + "parameters": [ + { + "name": "catalog", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CatalogName", + "description": "Catalog to view example queries for." + }, + "description": "Catalog to view example queries for." + }, + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "maxLength": 64, + "description": "Server-assigned job ID for the request", + "title": "Job Id" + }, + "description": "Server-assigned job ID for the request" + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "Body_TAPAsyncHandler_post__catalog__async_post": { + "properties": { + "LANG": { + "anyOf": [ + { + "$ref": "#/components/schemas/LangName" + }, + { + "type": "string" + } + ], + "title": "Lang", + "description": "Query language to be used for this request", + "default": "ADQL" + }, + "responseformat": { + "$ref": "#/components/schemas/EncodingEnum", + "description": "Content type of the response", + "default": "votable" + }, + "FORMAT": { + "$ref": "#/components/schemas/EncodingEnum", + "description": "Content type of the response", + "default": "votable" + }, + "QUERY": { + "type": "string", + "title": "Query", + "description": "Query to perform", + "default": "" + }, + "MAXREC": { + "type": "integer", + "minimum": 0.0, + "title": "Maxrec", + "description": "Maximum number of records to be returned (max 100k)", + "default": 100000 + }, + "RUNID": { + "anyOf": [ + { + "type": "string", + "maxLength": 64 + }, + { + "type": "null" + } + ], + "title": "Runid", + "description": "RUNID for the request", + "default": "" + }, + "PHASE": { + "anyOf": [ + { + "type": "string", + "enum": [ + "RUN" + ], + "const": "RUN" + }, + { + "type": "null" + } + ], + "title": "Phase", + "description": "Execution Phase (set to RUN for autorun)" + } + }, + "type": "object", + "title": "Body_TAPAsyncHandler_post__catalog__async_post" + }, + "Body_TAPAsyncHandler_post_update_job_destruction__catalog__async__job_id__destruction_post": { + "properties": { + "DESTRUCTION": { + "type": "string", + "format": "date-time", + "title": "Destruction", + "description": "ISO 8601 UTC datetime for proposed job destruction" + } + }, + "type": "object", + "title": "Body_TAPAsyncHandler_post_update_job_destruction__catalog__async__job_id__destruction_post" + }, + "Body_TAPAsyncHandler_post_update_job_execution_duration__catalog__async__job_id__executionduration_post": { + "properties": { + "EXECUTIONDURATION": { + "type": "integer", + "minimum": 0.0, + "title": "Executionduration", + "description": "Requested execution duration in seconds", + "default": 600 + } + }, + "type": "object", + "title": "Body_TAPAsyncHandler_post_update_job_execution_duration__catalog__async__job_id__executionduration_post" + }, + "Body_TAPAsyncHandler_post_update_job_info__catalog__async__job_id__post": { + "properties": { + "PHASE": { + "anyOf": [ + { + "$ref": "#/components/schemas/PhaseAction" + }, + { + "type": "null" + } + ], + "description": "Execution Phase" + }, + "DESTRUCTION": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "null" + } + ], + "title": "Destruction", + "description": "ISO 8601 UTC datetime for proposed job destruction" + }, + "ACTION": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Action", + "description": "Action to apply to job: Only DELETE supported." + } + }, + "type": "object", + "title": "Body_TAPAsyncHandler_post_update_job_info__catalog__async__job_id__post" + }, + "Body_TAPAsyncHandler_post_update_job_phase__catalog__async__job_id__phase_post": { + "properties": { + "PHASE": { + "$ref": "#/components/schemas/PhaseAction", + "description": "Execution Phase" + } + }, + "type": "object", + "required": [ + "PHASE" + ], + "title": "Body_TAPAsyncHandler_post_update_job_phase__catalog__async__job_id__phase_post" + }, + "Body_TAPAsyncHandler_update_job_info_parameters__catalog__async__job_id__parameters_post": { + "properties": { + "QUERY": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Query", + "description": "Query to be executed" + }, + "LANG": { + "anyOf": [ + { + "$ref": "#/components/schemas/LangName" + }, + { + "type": "null" + } + ], + "description": "Query langauge to be used." + }, + "RESPONSEFORMAT": { + "anyOf": [ + { + "$ref": "#/components/schemas/EncodingEnum" + }, + { + "type": "null" + } + ], + "description": "Content type of the response" + }, + "FORMAT": { + "anyOf": [ + { + "$ref": "#/components/schemas/EncodingEnum" + }, + { + "type": "null" + } + ], + "description": "Content type of the response" + }, + "MAXREC": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Maxrec", + "description": "Maximum number of records to return" + }, + "RUNID": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Runid", + "description": "Run ID to be used" + } + }, + "type": "object", + "title": "Body_TAPAsyncHandler_update_job_info_parameters__catalog__async__job_id__parameters_post" + }, + "Body_TAPHandler_post__catalog__sync_post": { + "properties": { + "LANG": { + "$ref": "#/components/schemas/LangName", + "description": "Query language to be used for this request" + }, + "responseformat": { + "$ref": "#/components/schemas/EncodingEnum", + "description": "Content type of the response", + "default": "votable" + }, + "format": { + "$ref": "#/components/schemas/EncodingEnum", + "description": "Content type of the response", + "default": "votable" + }, + "QUERY": { + "type": "string", + "title": "Query", + "description": "Query to perform" + }, + "MAXREC": { + "type": "integer", + "minimum": 0.0, + "title": "Maxrec", + "description": "Maximum number of records to be returned (max 100k)", + "default": 100000 + }, + "runid": { + "anyOf": [ + { + "type": "string", + "maxLength": 64 + }, + { + "type": "null" + } + ], + "title": "Runid", + "description": "RUNID for the request" + } + }, + "type": "object", + "required": [ + "LANG", + "QUERY" + ], + "title": "Body_TAPHandler_post__catalog__sync_post" + }, + "CatalogName": { + "type": "string", + "enum": [ + "caom", + "classy", + "gaiadr3", + "hsc", + "hscv2", + "mast_catalogs", + "missionmast", + "ps1dr1", + "ps1dr2", + "registry", + "roman_catalogs", + "skymapperdr4", + "tic", + "ullyses", + "goods", + "candels", + "3dhst", + "deepspace" + ], + "title": "CatalogName" + }, + "EncodingEnum": { + "type": "string", + "enum": [ + "json", + "csv", + "xml", + "votable" + ], + "title": "EncodingEnum", + "description": "Enumeration class for the format_param (format) query param" + }, + "ExecutionPhase": { + "type": "string", + "enum": [ + "PENDING", + "QUEUED", + "EXECUTING", + "COMPLETED", + "ERROR", + "UNKNOWN", + "HELD", + "SUSPENDED", + "ABORTED", + "ARCHIVED" + ], + "title": "ExecutionPhase", + "description": "Enumeration of possible phases of job execution." + }, + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "type": "array", + "title": "Detail" + } + }, + "type": "object", + "title": "HTTPValidationError" + }, + "LangName": { + "type": "string", + "enum": [ + "ADQL", + "ADQL-2.1" + ], + "title": "LangName", + "description": "Enum for language names as param" + }, + "PhaseAction": { + "type": "string", + "enum": [ + "ABORT", + "RUN" + ], + "title": "PhaseAction", + "description": "Enum for async job phase requests" + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "type": "array", + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + }, + "input": { + "title": "Input" + }, + "ctx": { + "type": "object", + "title": "Context" + } + }, + "type": "object", + "required": [ + "loc", + "msg", + "type" + ], + "title": "ValidationError" + } + } + }, + "servers": [ + { + "url": "/vo-tap/api/v0.1" + } + ] +} \ No newline at end of file diff --git a/astroquery/mast/tests/data/tap_columns.vot b/astroquery/mast/tests/data/tap_columns.vot new file mode 100644 index 0000000000..3a51181cdd --- /dev/null +++ b/astroquery/mast/tests/data/tap_columns.vot @@ -0,0 +1,915 @@ + + + + + + + + + + Column name + + + + + ADQL datatype + + + + + Unit in VO standard format + + + + + UCD of column if any + + + + + Brief description of column + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
idlong + meta.id;meta.maintess input catalog identifier
versionchar + meta.idcatalog version
hipint + meta.idhipparcos identifier
tycchar + meta.idtycho2 identifier
ucacchar + meta.iducac4 identifier
twomasschar + meta.id2mass identifier (hhmmssss+ddmmsss j2000)
sdsslong + meta.idsdss dr9 objid identifier
allwisechar + meta.idallwise identifier (jhhmmss.ss+ddmmss.s)
gaiachar + meta.idgaia dr2 identifier
apasschar + meta.idapass dr9 identifier
kicint + meta.idkic identifier
objtypechar + src.class.stargalaxyobject type (star or extended)
typesrcchar + meta.refthe source of the object in the tic
radoubledegpos.eq.ra;meta.mainright ascension (j2000)
decdoubledegpos.eq.dec;meta.maindeclination (j2000)
posflagchar + meta.ref;pos.framesource of the position
pmrafloatmas/yrpos.pm;pos.eq.raproper motion in right ascension
e_pmrafloatmas/yrstat.error;pos.pm;pos.eq.rauncertainty in pmra
pmdecfloatmas/yrpos.pm;pos.eq.decproper motion in declination
e_pmdecfloatmas/yrstat.error;pos.pm;pos.eq.decuncertainty in pmde
pmflagchar + meta.ref;pos.framesource of the proper motion
plxfloatmaspos.parallaxparallax
e_plxfloatmasstat.error;pos.parallaxerror in the parallax
parflagchar + meta.ref;pos.framesource of the parallax
gallongdoubledegpos.galactic.longalactic longitude
gallatdoubledegpos.galactic.latgalactic latitude
eclongdoubledegpos.ecliptic.lonecliptic longitude
eclatdoubledegpos.ecliptic.latecliptic latitude
bmagfloatmagphot.mag;em.opt.bjohnson b magnitude
e_bmagfloatmagstat.error;phot.maguncertainty in bmag
vmagfloatmagphot.mag;em.opt.vjohnson v magnitude
e_vmagfloatmagstat.error;phot.maguncertainty in vmag
umagfloatmagphot.mag;em.opt.usdss u-band (ab) magnitude
e_umagfloatmagstat.error;phot.maguncertainty in umag
gmagfloatmagphot.mag;em.opt.bsdss g-band (ab) magnitude
e_gmagfloatmagstat.error;phot.maguncertainty in gmag
rmagfloatmagphot.mag;em.opt.rsdss r-band (ab) magnitude
e_rmagfloatmagstat.error;phot.maguncertainty in rmag
imagfloatmagphot.mag;em.opt.isdss i-band (ab) magnitude
e_imagfloatmagstat.error;phot.maguncertainty in imag
zmagfloatmagphot.mag;em.opt.isdss z-band (ab) magnitude
e_zmagfloatmagstat.error;phot.maguncertainty in zmag
jmagfloatmagphot.mag;em.ir.j2mass johnson j-band magnitude
e_jmagfloatmagstat.error;phot.maguncertainty in jmag
hmagfloatmagphot.mag;em.ir.h2mass johnson h-band magnitude
e_hmagfloatmagstat.error;phot.maguncertainty in hmag
kmagfloatmagphot.mag;em.ir.k2mass johnson k-band magnitude
e_kmagfloatmagstat.error;phot.maguncertainty in kmag
twomflagchar + meta.code.qualquality flags for 2mass
proxfloatarcsecpos.angdistanceobject proximity
w1magfloatmagphot.mag;em.ir.3-4umallwise w1 (3.4um) magnitude
e_w1magfloatmagstat.error;phot.maguncertainty in w1mag
w2magfloatmagphot.mag;em.ir.4-8umallwise w2 (4.6um) magnitude
e_w2magfloatmagstat.error;phot.maguncertainty in w2mag
w3magfloatmagphot.mag;em.ir.8-15umallwise w3 (12um) magnitude
e_w3magfloatmagstat.error;phot.maguncertainty in w3mag
w4magfloatmagphot.mag;em.ir.15-30umallwise w4 (22um) magnitude
e_w4magfloatmagstat.error;phot.maguncertainty in w4mag
gaiamagfloatmagphot.mag;em.opt.vgaiadr2 g magnitude
e_gaiamagfloatmagstat.error;phot.maguncertainty in gmag
tmagfloatmagphot.mag;em.opttess magnitude
e_tmagfloatmagstat.error;phot.maguncertainty in tess mag
tessflagchar + meta.codetess magnitude flag
spflagchar + meta.codestellar properties flag
tefffloatKphys.temperature.effectiveeffective temperature
e_tefffloatKstat.error;phys.temperature.effectiveuncertainty in teff
loggfloatcm/s**2phys.gravitylog of the surface gravity
e_loggfloatcm/s**2stat.error;phys.gravityuncertainty in logg
mhfloat + phys.abund.zmetallicity [m/h]
e_mhfloat + stat.error;phys.abund.zuncertainty in [m/h]
radfloatsolRadphys.size.radiusradius
e_radfloatsolRadstat.error;phys.size.radiusuncertainty in radius
massfloatsolMassphys.massmass
e_massfloatsolMassstat.error;phys.massuncertainty in mass
rhofloat + phys.densitystellar density in solar units (mass/rad^3)
e_rhofloat + stat.error;phys.densityuncertainty in rho
lumclasschar + src.class.luminosityluminosity class
lumfloatsolLumphys.luminositystellar luminosity; rad^2*(teff/5772)^4
e_lumfloatsolLumstat.error;phys.luminosityuncertainty in luminosity
dfloatpcpos.distance;pos.heliocentricdistance
e_dfloatpcstat.error;pos.distanceuncertainty in distance
ebvfloatmagphot.color.excessapplied color excess
e_ebvfloatmagstat.error;phot.color.excessuncertainty in e(b-v)
numcontint + meta.numbernumber of contaminants found within 10arcsec of the star used in the calculation of the contamination ratio
contratiofloat + stat.fitcontamination ratio
dispositionchar + meta.code.classdisposition type
duplicate_idlong + meta.code.multiptic id of another object in duplicate or split set of stars
priorityfloat + meta.numberpriority (number from 0 to 1 = highest priority)
eneg_ebvfloatmagstat.error;phot.color.excess;stat.minnegative error for e(b-v)
epos_ebvfloatmagstat.error;phot.color.excess;stat.maxpositive error for e(b-v)
ebvflagchar + meta.ref;phot.color.excesssource of e(b-v)
eneg_massfloatsolMassstat.error;phys.mass;stat.minnegative error for mass
epos_massfloatsolMassstat.error;phys.mass;stat.maxpositive error for mass
eneg_radfloatsolRadstat.error;phys.size.radius;stat.minnegative error for radius
epos_radfloatsolRadstat.error;phys.size.radius;stat.maxpositive error for radius
eneg_rhofloat + stat.error;phys.density;stat.minnegative error for stellar density
epos_rhofloat + stat.error;phys.density;stat.maxpositive error for stellar density
eneg_loggfloatcm/s**2stat.error;phys.gravity;stat.minnegative error for surface gravity
epos_loggfloatcm/s**2stat.error;phys.gravity;stat.maxpositive error for surface gravity
eneg_lumfloatsolLumstat.error;phys.luminosity;stat.minnegative error for luminosity
epos_lumfloatsolLumstat.error;phys.luminosity;stat.maxpositive error for luminosity
eneg_distfloatpcstat.error;pos.distance;stat.minnegative error for distance
epos_distfloatpcstat.error;pos.distance;stat.maxpositive error for distance
distflagchar + meta.ref;pos.distancesource for distance
eneg_tefffloatKstat.error;phys.temperature.effective;stat.minnegative error for effective temperature
epos_tefffloatKstat.error;phys.temperature.effective;stat.maxpositive error for effective temperature
teffflagchar + meta.ref;phys.temperature.effectivesource for effective temperature
gaiabpfloatmagphot.mag;em.opt.bgaiadr2 bp magnitude
e_gaiabpfloatmagstat.error;phot.maguncertainty in bp magnitude
gaiarpfloatmagphot.mag;em.opt.rgaiadr2 rp magnitude
e_gaiarpfloatmagstat.error;phot.maguncertainty in rp magnitude
gaiaqflagint + meta.code.qualquality flags for gaia information
starchareflagchar + stat.errorasymmetric errors
vmagflagchar + meta.ref;phot.mag;em.opt.vsource of v magnitude
bmagflagchar + meta.ref;phot.mag;em.opt.bsource of b magnitude
splistschar + meta.codeidentifies if star is in a specially curated list
e_radoublemasstat.error;pos.eq.raerror in radeg
e_decdoublemasstat.error;pos.eq.decerror in decgeg
ra_origdoubledegpos.eq.rara from original catalog
dec_origdoubledegpos.eq.decdec from original catalog
e_ra_origdoublemasstat.error;pos.eq.raerror in ra_orig
e_dec_origdoublemasstat.error;pos.eq.decerror in dec_orig
raddflagint + meta.codedwarf by radius; 0: giant by radius; -1: insufficient information
wdflagint + meta.codestar in gaia photometric white dwarf region
objidlong + meta.recorddatabase internal identifier
+ + diff --git a/astroquery/mast/tests/data/tap_results.vot b/astroquery/mast/tests/data/tap_results.vot new file mode 100644 index 0000000000..9900f6c70f --- /dev/null +++ b/astroquery/mast/tests/data/tap_results.vot @@ -0,0 +1,895 @@ + + + + + + + + + + tess input catalog identifier + + + + + catalog version + + + + + hipparcos identifier + + + + + tycho2 identifier + + + + + ucac4 identifier + + + + + 2mass identifier (hhmmssss+ddmmsss j2000) + + + + + sdss dr9 objid identifier + + + + + allwise identifier (jhhmmss.ss+ddmmss.s) + + + + + gaia dr2 identifier + + + + + apass dr9 identifier + + + + + kic identifier + + + + + object type (star or extended) + + + + + the source of the object in the tic + + + + + right ascension (j2000) + + + + + declination (j2000) + + + + + source of the position + + + + + proper motion in right ascension + + + + + uncertainty in pmra + + + + + proper motion in declination + + + + + uncertainty in pmde + + + + + source of the proper motion + + + + + parallax + + + + + error in the parallax + + + + + source of the parallax + + + + + galactic longitude + + + + + galactic latitude + + + + + ecliptic longitude + + + + + ecliptic latitude + + + + + johnson b magnitude + + + + + uncertainty in bmag + + + + + johnson v magnitude + + + + + uncertainty in vmag + + + + + sdss u-band (ab) magnitude + + + + + uncertainty in umag + + + + + sdss g-band (ab) magnitude + + + + + uncertainty in gmag + + + + + sdss r-band (ab) magnitude + + + + + uncertainty in rmag + + + + + sdss i-band (ab) magnitude + + + + + uncertainty in imag + + + + + sdss z-band (ab) magnitude + + + + + uncertainty in zmag + + + + + 2mass johnson j-band magnitude + + + + + uncertainty in jmag + + + + + 2mass johnson h-band magnitude + + + + + uncertainty in hmag + + + + + 2mass johnson k-band magnitude + + + + + uncertainty in kmag + + + + + quality flags for 2mass + + + + + object proximity + + + + + allwise w1 (3.4um) magnitude + + + + + uncertainty in w1mag + + + + + allwise w2 (4.6um) magnitude + + + + + uncertainty in w2mag + + + + + allwise w3 (12um) magnitude + + + + + uncertainty in w3mag + + + + + allwise w4 (22um) magnitude + + + + + uncertainty in w4mag + + + + + gaiadr2 g magnitude + + + + + uncertainty in gmag + + + + + tess magnitude + + + + + uncertainty in tess mag + + + + + tess magnitude flag + + + + + stellar properties flag + + + + + effective temperature + + + + + uncertainty in teff + + + + + log of the surface gravity + + + + + uncertainty in logg + + + + + metallicity [m/h] + + + + + uncertainty in [m/h] + + + + + radius + + + + + uncertainty in radius + + + + + mass + + + + + uncertainty in mass + + + + + stellar density in solar units (mass/rad^3) + + + + + uncertainty in rho + + + + + luminosity class + + + + + stellar luminosity; rad^2*(teff/5772)^4 + + + + + uncertainty in luminosity + + + + + distance + + + + + uncertainty in distance + + + + + applied color excess + + + + + uncertainty in e(b-v) + + + + + number of contaminants found within 10arcsec of the star used in + the calculation of the contamination ratio + + + + + contamination ratio + + + + + disposition type + + + + + tic id of another object in duplicate or split set of stars + + + + + priority (number from 0 to 1 = highest priority) + + + + + negative error for e(b-v) + + + + + positive error for e(b-v) + + + + + source of e(b-v) + + + + + negative error for mass + + + + + positive error for mass + + + + + negative error for radius + + + + + positive error for radius + + + + + negative error for stellar density + + + + + positive error for stellar density + + + + + negative error for surface gravity + + + + + positive error for surface gravity + + + + + negative error for luminosity + + + + + positive error for luminosity + + + + + negative error for distance + + + + + positive error for distance + + + + + source for distance + + + + + negative error for effective temperature + + + + + positive error for effective temperature + + + + + source for effective temperature + + + + + gaiadr2 bp magnitude + + + + + uncertainty in bp magnitude + + + + + gaiadr2 rp magnitude + + + + + uncertainty in rp magnitude + + + + + quality flags for gaia information + + + + + asymmetric errors + + + + + source of v magnitude + + + + + source of b magnitude + + + + + identifies if star is in a specially curated list + + + + + error in radeg + + + + + error in decgeg + + + + + ra from original catalog + + + + + dec from original catalog + + + + + error in ra_orig + + + + + error in dec_orig + + + + + dwarf by radius; 0: giant by radius; -1: insufficient information + + + + + star in gaia photometric white dwarf region + + + + + database internal identifier + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
42770553920190415 + + 382-12368718475108-1342233 + J184751.04-134222.641051754443095892486823646 + STARgaia2281.9629072438-13.7065985389427gaia2-5.661380.0764805-1.146080.0688171gaia20.3980230.0440811gaia220.2937183862185-5.42345791126297281.7724783304729.2451314111868416.8560.18115.3230.149 + + + + + + + + + + 11.9420.04711.2480.06810.7520.035EEE-222-111-000-0-0 + 10.3340.02310.4480.02110.6720.0889.134 + 14.71930.00047613.67730.0106reredgaia24418123 + + + + + + + + + + + + + 2360.4257.6050.5037430.0162428 + + SPLIT + + 0.01257280.0199128panstarrs + + + + + + + + + + 230.46284.75bj2018 + + dered15.77360.00978613.67780.0077211 + ucac4bpbj + 1.194495727858711.06728505482206281.962882153899-13.70660347345510.03513726893589260.03637388286974680022205941
60699594820190415 + + + + + + 510528438770827136 + + STARgaia219.415533913500161.0286394857995gaia2-0.9694640.1876690.8473730.18527gaia20.3482330.134513gaia2126.103557179616-1.6815130899793647.338495783371947.6142191319478 + + 18.58020.0486 + + + + + + + + + + + + + + + + + + + + + + + + + + 18.16120.00123517.38710.0271reredgaia25722154 + + + + + + + + + + + + + 2527.82921.4750.5078120.03941955 + + DUPLICATE54375580 + 0.02864960.0501895panstarrs + + + + + + + + + + 665.591177.36bj2018 + + dered18.75960.02656117.25690.0143231 + gaia2 + + 3.378239895504092.8735447568218719.415525295995661.02864313421130.1081048093514390.1033669687738271021716702
+ + diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index 105582fd9a..7d3703c359 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -4,25 +4,52 @@ import os import re import warnings +from datetime import datetime +from pathlib import Path from shutil import copyfile from unittest.mock import MagicMock, patch -from pathlib import Path import astropy.units as u -import pytest import numpy as np -from astropy.table import Table, unique -from astropy.coordinates import SkyCoord +import pytest +from astropy.coordinates import Angle, SkyCoord from astropy.io import fits +from astropy.io.votable import parse +from astropy.table import Table, unique +from astropy.time import Time from astropy.utils.exceptions import AstropyDeprecationWarning +from pyvo.dal import TAPResults +from pyvo.dal.exceptions import DALQueryError +from pyvo.io.vosi import parse_capabilities from requests import HTTPError, Response -from astroquery.mast import (Catalogs, MastMissions, Observations, Tesscut, Zcut, Mast, utils, services, - discovery_portal, auth, core, cloud) +from astroquery.exceptions import ( + BlankResponseWarning, + InputWarning, + InvalidQueryError, + MaxResultsWarning, + NoResultsWarning, + RemoteServiceError, + ResolverError, +) +from astroquery.mast import ( + Catalogs, + Mast, + MastMissions, + Observations, + Tesscut, + Zcut, + auth, + cloud, + core, + discovery_portal, + services, + utils, +) from astroquery.mast.cloud import CloudAccess from astroquery.utils.mocks import MockResponse -from astroquery.exceptions import (BlankResponseWarning, InvalidQueryError, InputWarning, MaxResultsWarning, - NoResultsWarning, RemoteServiceError, ResolverError) + +from ..catalog_collection import DEFAULT_CATALOGS, CatalogCollection, CatalogMetadata try: # Optional dependency import for cloud access functionality @@ -30,6 +57,13 @@ except ImportError: pass +try: + # Optional dependency import for region handling in collections queries + from regions import CirclePixelRegion, CircleSkyRegion, PixCoord, PolygonSkyRegion + HAS_REGIONS = True +except ImportError: + HAS_REGIONS = False + DATA_FILES = {'Mast.Caom.Cone': 'caom.json', 'Mast.Name.Lookup': 'resolver.json', 'mission_search_results': 'mission_results.json', @@ -61,6 +95,11 @@ 'get_cloud_paths': 'mast_relative_path.json', 'panstarrs': 'panstarrs.json', 'panstarrs_columns': 'panstarrs_columns.json', + 'tap_collections': 'tap_collections.json', # Collections available + 'tap_catalogs': 'tap_catalogs.vot', # Catalogs for TIC + 'tap_columns': 'tap_columns.vot', # Column metadata + 'tap_capabilities': 'tap_capabilities.xml', # TAP service capabilities + 'tap_results': 'tap_results.vot', # Results of a TAP query 'tess_cutout': 'astrocut_107.27_-70.0_5x5.zip', 'tess_sector': 'tess_sector.json', 'z_cutout_fit': 'astrocut_189.49206_62.20615_100x100px_f.zip', @@ -81,6 +120,10 @@ def patch_post(request): mp.setattr(discovery_portal.PortalAPI, '_request', post_mockreturn) mp.setattr(services.ServiceAPI, '_request', service_mockreturn) mp.setattr(auth.MastAuth, 'session_info', session_info_mockreturn) + mp.setattr( + 'astroquery.mast.catalog_collection.TAPService', + lambda *args, **kwargs: vo_tap_mock() + ) mp.setattr(Tesscut, '_download_file', tesscut_download_mockreturn) mp.setattr(Zcut, '_download_file', zcut_download_mockreturn) @@ -89,6 +132,33 @@ def patch_post(request): return mp +@pytest.fixture +def patch_tap(request, reset_catalogs_cache): + mp = request.getfixturevalue("monkeypatch") + + mock_tap = vo_tap_mock() + mp.setattr( + 'astroquery.mast.catalog_collection.TAPService', + lambda *args, **kwargs: mock_tap + ) + # We have to set this because CatalogsClass uses a simple request to get collections + mp.setattr(utils, '_simple_request', request_mockreturn) + + return mock_tap + + +@pytest.fixture +def reset_catalogs_cache(): + Catalogs._collections_cache.clear() + yield + + +def get_patch_tap_query(patch_tap): + args, _ = patch_tap.run_sync.call_args + query = args[0] + return query + + @pytest.fixture() def patch_boto3(monkeypatch, reset_cloud_state): """Fixture to patch boto3 client and resource for cloud access tests.""" @@ -178,6 +248,8 @@ def request_mockreturn(url, params={}): filename = data_path(DATA_FILES['panstarrs_columns']) elif 'path_lookup' in url: filename = data_path(DATA_FILES['get_cloud_paths']) + elif 'vo-tap' in url: + filename = data_path(DATA_FILES['tap_collections']) with open(filename, 'rb') as infile: content = infile.read() return MockResponse(content) @@ -236,12 +308,49 @@ def zcut_download_mockreturn(url, file_path): return +def vo_tap_mock(): + def run_sync_mock(query, **kwargs): + if 'invalid' in query: + # Use this when wanting to simulate a DALQueryError + # Where it occurs will depend on where you pass it (collection, catalog, parameter, etc.) + raise DALQueryError('Simulated TAP query error for testing.') + elif 'tap_schema.tables' in query: + # Queries to get catalogs + filename = data_path(DATA_FILES['tap_catalogs']) + elif 'tap_schema.columns' in query: + # Queries to get column metadata + filename = data_path(DATA_FILES['tap_columns']) + elif 'WHERE' in query: + # Queries with results, keep in mind this is not meaningful and results won't match the query + filename = data_path(DATA_FILES['tap_results']) + votable = parse(filename) + + if 'empty' in query: + # Simulate a query that returns no results by clearing the resources in the votable + for resource in votable.resources: + for table in resource.tables: + table.array = np.array([]) # Clear the data to simulate no results + + return TAPResults(votable) + + # Mock TAPService + mock_tap = MagicMock() + mock_tap.run_sync.side_effect = run_sync_mock + + # Capabilities -> Not much to do here + filename = data_path(DATA_FILES['tap_capabilities']) + with open(filename, "rb") as f: + caps = parse_capabilities(f) + mock_tap.capabilities = caps + + return mock_tap + ########################### # MissionSearchClass Test # ########################### -def test_missions_query_region_async(): +def test_missions_query_region_async(patch_post): responses = MastMissions.query_region_async(regionCoords, radius=0.002, sci_pi_last_name='GORDON') assert isinstance(responses, MockResponse) @@ -1355,171 +1464,1012 @@ def test_observations_disable_cloud_dataset(patch_boto3): assert Observations._cloud_enabled_explicitly is False -###################### -# CatalogClass tests # -###################### +####################### +# CatalogsClass tests # +####################### -def test_catalogs_query_region_async(): - responses = Catalogs.query_region_async(regionCoords, radius=0.002) - assert isinstance(responses, list) +def test_catalogs_attributes(patch_tap): + Catalogs.query_criteria( + collection="tic", + region="Circle ICRS 202.4656816 +47.1999842 0.04", + radius=0.002 * u.deg, + offset=1, + sort_by="ra", + ) + # Should not change after query + assert Catalogs.collection == "hsc" + assert Catalogs.catalog == "dbo.SumMagAper2CatView" -def test_catalogs_fabric_query_region_async(): - responses = Catalogs.query_region_async(regionCoords, radius=0.002, catalog="panstarrs", table="mean") - assert isinstance(responses, MockResponse) +def test_catalogs_get_catalogs(patch_tap): + catalogs = Catalogs.get_catalogs("tic") + assert isinstance(catalogs, Table) -def test_catalogs_query_region(): - result = Catalogs.query_region(regionCoords, radius=0.002 * u.deg) - assert isinstance(result, Table) +def test_catalogs_get_column_metadata(patch_tap): + metadata = Catalogs.get_column_metadata(collection="tic") + assert isinstance(metadata, Table) - result = Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="hsc", version=2) - assert isinstance(result, Table) - with pytest.warns(InputWarning) as i_w: - Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="hsc", version=5) - assert "Invalid HSC version number" in str(i_w[0].message) +def test_catalogs_query_criteria(patch_tap): + # Coordinates + result = Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + radius=0.002 * u.deg, + limit=2, + ) - result = Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="galex") assert isinstance(result, Table) + assert len(result) > 0 + assert "dec" in result.colnames + query = get_patch_tap_query(patch_tap) + assert "FROM dbo.catalogrecord" in query + assert "CONTAINS" in query + assert "CIRCLE" in query + assert "TOP 2" in query + + # Region query + result = Catalogs.query_criteria( + collection="tic", + region="Circle ICRS 202.4656816 +47.1999842 0.04", + radius=0.002 * u.deg, + offset=1, + sort_by="ra", + ) - result = Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="gaia", version=2) assert isinstance(result, Table) - - result = Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="gaia", version=1) + assert len(result) > 0 + assert "dec" in result.colnames + query = get_patch_tap_query(patch_tap) + assert "FROM dbo.catalogrecord" in query + assert "CONTAINS" in query + assert "CIRCLE" in query + assert "OFFSET" in query + assert "ORDER BY" in query + + # Non-positional query + result = Catalogs.query_criteria( + collection="tic", + filters={"pmra": [">-10", "<10"]}, + ) assert isinstance(result, Table) + assert len(result) > 0 + assert "dec" in result.colnames + query = get_patch_tap_query(patch_tap) + assert "FROM dbo.catalogrecord" in query + assert "WHERE" in query + assert "pmra > -10" in query + assert "pmra < 10" in query + + +def test_catalogs_invalid_query_criteria(patch_tap): + # Specifying both region and coordinates + with pytest.raises(InvalidQueryError, match="Specify either `region` or `coordinates`"): + Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + region="Circle ICRS 202.4656816 +47.1999842 0.04" + ) - with pytest.warns(InputWarning) as i_w: - Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="gaia", version=5) - assert "Invalid Gaia version number" in str(i_w[0].message) + # Specifying both region and object_name + with pytest.raises(InvalidQueryError, match="Specify either `region` or `object_name`"): + Catalogs.query_criteria( + collection="tic", + object_name="M31", + region="Circle ICRS 202.4656816 +47.1999842 0.04" + ) - result = Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="Sample") - assert isinstance(result, Table) + # Named parameters and filters dict specifying criteria + with pytest.raises(InvalidQueryError, match="Criteria specified both"): + Catalogs.query_criteria( + collection="tic", + object_name="M31", + file_suffix=['A', 'B', '!C'], + filters={"file_suffix": ['A', 'B', '!C']} + ) + # sort_by cols and sort_desc different lengths + with pytest.raises(InvalidQueryError, match="must be 1 or equal to length of 'sort_by'"): + Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + sort_by=["ra", "dec"], + sort_desc=[True, False, True] + ) -def test_catalogs_fabric_query_region(): - result = Catalogs.query_region(regionCoords, radius=0.002 * u.deg, catalog="panstarrs", table="mean") - assert isinstance(result, Table) + # Invalid sort col + with pytest.raises(InvalidQueryError, match="Filter 'fake' is not recognized"): + Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + sort_by="fake", + ) + # Invalid filter + with pytest.raises(InvalidQueryError, match="Filter 'fake' is not recognized"): + Catalogs.query_criteria( + collection="tic", + fake=1 + ) -def test_catalogs_query_object_async(): - responses = Catalogs.query_object_async("M101", radius="0.002 deg") - assert isinstance(responses, list) + # Collection is not a string + with pytest.raises(InvalidQueryError, match="Collection name must be a string."): + Catalogs.query_criteria( + collection=123, + objtype="star" + ) + # Warn if result table is empty + with pytest.warns(NoResultsWarning, match="The query returned no results."): + Catalogs.query_criteria( + collection="tic", + objtype="empty" + ) -def test_catalogs_fabric_query_object_async(): - responses = Catalogs.query_object_async("M101", radius="0.002 deg", catalog="panstarrs", table="mean") - assert isinstance(responses, MockResponse) +def test_catalogs_query_region(patch_tap): + # Passing region coords and radius + result = Catalogs.query_region( + regionCoords, + radius=0.002 * u.deg, + collection="tic" + ) -def test_catalogs_query_object(): - result = Catalogs.query_object("M101", radius=".002 deg") assert isinstance(result, Table) + assert len(result) > 0 + assert "ra" in result.colnames + query = get_patch_tap_query(patch_tap) + assert "FROM dbo.catalogrecord" in query + assert "CONTAINS" in query + assert "CIRCLE" in query + assert "POINT" in query + + +def test_catalogs_invalid_query_region(): + # Query without region or coordinates + with pytest.raises(InvalidQueryError, match="Must specify either `region` or `coordinates`"): + Catalogs.query_region( + collection="tic", + ) + + # Query with unsupported region type + with pytest.raises(InvalidQueryError, match="does not support ADQL region type 'POLYGON'"): + Catalogs.query_region( + region="Polygon ICRS 202.4656816 +47.1999842 202.5656816 +47.2999842 202.3656816 +47.0999842", + collection="tic" + ) -def test_catalogs_fabric_query_object(): - result = Catalogs.query_object("M101", radius=".002 deg", catalog="panstarrs", table="mean") +def test_catalogs_query_object(patch_tap): + # Object and radius query + radius = .001 + result = Catalogs.query_object( + "M10", + radius=radius, + collection="TIC" + ) + assert isinstance(result, Table) + assert len(result) > 0 + assert "ra" in result.colnames + query = get_patch_tap_query(patch_tap) + assert "FROM dbo.catalogrecord" in query + assert "CONTAINS" in query + assert "POINT" in query + assert str(radius) in query + + +def test_catalogs_init_with_catalog(patch_tap): + catalog = Catalogs( + collection="tic", + catalog="tap_schema.schemas" + ) + assert catalog.catalog == "tap_schema.schemas" -def test_catalogs_query_criteria_async(): - responses = Catalogs.query_criteria_async(catalog="Tic", - Bmag=[30, 50], objType="STAR") - assert isinstance(responses, list) +def test_catalogs_setting_catalog(patch_tap): + catalog = Catalogs( + collection="tic", + catalog="tap_schema.schemas" + ) + catalog.catalog = "tap_schema.key_columns" + assert catalog.catalog == "tap_schema.key_columns" - responses = Catalogs.query_criteria_async(catalog="Ctl", - Bmag=[30, 50], objType="STAR") - assert isinstance(responses, list) - responses = Catalogs.query_criteria_async(catalog="Tic", object_name="M10", - Bmag=[30, 50], objType="STAR") - assert isinstance(responses, list) +def test_catalogs_get_collections_cached(patch_tap): + catalog = Catalogs("tic") + collections = catalog.get_collections() - responses = Catalogs.query_criteria_async(catalog="DiskDetective", - object_name="M10", radius=2, - state="complete") - assert isinstance(responses, list) + assert isinstance(collections, Table) + assert len(collections) > 0 + assert "collection_name" in collections.colnames - responses = Catalogs.query_criteria_async(catalog="panstarrs", object_name="M10", radius=2, - table="mean", qualityFlag=48) - assert isinstance(responses, MockResponse) - with pytest.raises(InvalidQueryError) as invalid_query: - Catalogs.query_criteria_async(catalog="Tic") - assert "non-positional" in str(invalid_query.value) +def test_catalogs_supports_spatial_queries(patch_tap): + catalog = Catalogs() + result = catalog.supports_spatial_queries( + collection="tic", + catalog="tap_schema.schemas" + ) - with pytest.raises(InvalidQueryError) as invalid_query: - Catalogs.query_criteria_async(catalog="SampleFail") - assert "query not available" in str(invalid_query.value) + assert isinstance(result, bool) + assert result + + +def test_catalogs_verify_collection(patch_tap): + valid = Catalogs._verify_collection("tic") + assert valid.lower() == "tic" + + # Renamed collection + renamed = list(Catalogs._renamed_collections.keys())[0] + new_name = Catalogs._renamed_collections[renamed] + with pytest.warns(InputWarning, match="has been renamed"): + result = Catalogs._verify_collection(renamed) + assert result == new_name + + # Invalid collection + with pytest.raises(InvalidQueryError, match="is not recognized"): + Catalogs._verify_collection("FAKE") + + # No longer supported collection + if Catalogs._no_longer_supported_collections: + unsupported = list(Catalogs._no_longer_supported_collections)[0] + with pytest.raises(InvalidQueryError) as exc: + Catalogs._verify_collection(unsupported) + assert "no longer supported" in str(exc.value) + + +def test_catalogs_parse_inputs(patch_tap): + collection_name = Catalogs.available_collections[0] + collection_obj, catalog = Catalogs._parse_inputs(collection=collection_name, catalog=None) + assert isinstance(collection_obj, CatalogCollection) + assert collection_obj.name == collection_name + assert catalog == collection_obj.default_catalog + + # Catalog parameter warning + with pytest.warns(DeprecationWarning, match="via the `catalog` parameter is deprecated."): + collection_name = Catalogs.available_collections[0] + collection_obj, catalog = Catalogs._parse_inputs(collection=None, catalog=collection_name) + assert isinstance(collection_obj, CatalogCollection) + assert catalog == collection_obj.default_catalog + + # Use catalog attribute if valid for collection + Catalogs.catalog = "dbo.catalogrecord" + collection_obj, catalog = Catalogs._parse_inputs(collection="tic") + assert collection_obj.name == "tic" + assert catalog == "dbo.catalogrecord" + + +def test_catalogs_parse_select_cols(patch_tap): + catalog = Catalogs("tic") + column_metadata = catalog.get_column_metadata() + result = Catalogs._parse_select_cols( + ["ra", "dec"], + column_metadata) + assert result == "ra, dec" + + # Close match suggestion + close_match_col = "gaiagaiabp" + with pytest.warns(InputWarning, match=" not found in catalog. Did you mean"): + result = Catalogs._parse_select_cols( + ["ra", close_match_col], + column_metadata + ) - with pytest.raises(InvalidQueryError) as invalid_query: - Catalogs.query_criteria_async(catalog="panstarrs", object_name="M10", coordinates=regionCoords, - objType="STAR") - assert "one of object_name and coordinates" in str(invalid_query.value) + # Empty columns + with pytest.raises(InvalidQueryError, match="No valid columns specified in `select_cols`"): + result = Catalogs._parse_select_cols( + [], + column_metadata + ) -def test_catalogs_query_criteria(): - # without position - result = Catalogs.query_criteria(catalog="Tic", - Bmag=[30, 50], objType="STAR") +def test_catalogs_parse_legacy_pagination(patch_tap): + catalog = Catalogs("tic") + limit, offset = catalog._parse_legacy_pagination( + limit=5000, + offset=0, + pagesize=10, + page=None, + ) + assert limit == 10 + assert offset == 0 + + # Missing pagesize + with pytest.warns(InputWarning, match="The 'page' parameter is ignored without 'pagesize'."): + catalog._parse_legacy_pagination( + limit=5000, + offset=0, + pagesize=None, + page=2, + ) - assert isinstance(result, Table) - result = Catalogs.query_criteria(catalog="Ctl", - Bmag=[30, 50], objType="STAR") +def test_catalogs_create_adql_region(patch_tap): + # String regions + adql_region = Catalogs._create_adql_region( + region="Circle 202.4656816 +47.1999842 0.2" + ) + assert adql_region == "CIRCLE('ICRS',202.4656816,+47.1999842,0.2)" - assert isinstance(result, Table) + adql_region = Catalogs._create_adql_region( + region="Polygon ICRS 202.4656816 +47.1999842 202.5656816 +47.2999842 202.3656816 +47.0999842" + ) + assert adql_region == "POLYGON('ICRS',202.4656816,+47.1999842,202.5656816,+47.2999842,202.3656816,+47.0999842)" - # with position - result = Catalogs.query_criteria(catalog="DiskDetective", - object_name="M10", radius=2, - state="complete") - assert isinstance(result, Table) + adql_region = Catalogs._create_adql_region( + region="Polygon 202.4656816 +47.1999842 202.5656816 +47.2999842 202.3656816 +47.0999842" + ) + assert adql_region == "POLYGON('ICRS',202.4656816,+47.1999842,202.5656816,+47.2999842,202.3656816,+47.0999842)" + + # Iterable coord pairs + adql_region = Catalogs._create_adql_region( + region=[ + (57.376, 24.053), + (56.391, 24.622), + (56.025, 24.049), + (56.616, 24.291) + ] + ) + assert adql_region == "POLYGON('ICRS',57.376,24.053,56.391,24.622,56.025,24.049,56.616,24.291)" - with pytest.raises(InvalidQueryError) as invalid_query: - Catalogs.query_criteria(catalog="Tic", object_name="M10") - assert "non-positional" in str(invalid_query.value) + if HAS_REGIONS: + # Astropy region objects + cone_region = CircleSkyRegion( + center=SkyCoord(10.8, 6.5, unit="deg"), + radius=Angle(0.5, unit="deg") + ) + adql_region = Catalogs._create_adql_region(region=cone_region) + assert adql_region == "CIRCLE('ICRS',10.8,6.5,0.5)" + + polygon_region = PolygonSkyRegion( + SkyCoord( + [57.376, 56.391, 56.025, 56.616], + [24.053, 24.622, 24.049, 24.291], + frame="icrs", + unit="deg", + ) + ) + adql_region = Catalogs._create_adql_region(region=polygon_region) + assert adql_region == "POLYGON('ICRS',57.376,24.053,56.391,24.622,56.025,24.049,56.616,24.291)" -def test_catalogs_query_hsc_matchid_async(): - responses = Catalogs.query_hsc_matchid_async(82371983) - assert isinstance(responses, list) +def test_catalogs_invalid_create_adql_region(patch_tap): + # Polygon without points + with pytest.raises(InvalidQueryError, match="Invalid POLYGON region string"): + Catalogs._create_adql_region(region="Polygon ICRS") - responses = Catalogs.query_hsc_matchid_async(82371983, version=2) - assert isinstance(responses, list) + # Polygon without sufficient points + with pytest.raises(InvalidQueryError, match="Invalid POLYGON region string"): + Catalogs._create_adql_region(region="Polygon ICRS 202.4656816 +47.1999842 202.5656816 +47.2999842") + + # Missing circle spec, frame not specified + with pytest.raises(InvalidQueryError, match="Invalid CIRCLE region string"): + Catalogs._create_adql_region(region="CIRCLE 202.4656816 +47.1999842") + + # Missing circle spec, frame specified + with pytest.raises(InvalidQueryError, match="Invalid CIRCLE region string"): + Catalogs._create_adql_region(region="CIRCLE ICRS 202.4656816 +47.1999842") + + # Invalid region str + with pytest.raises(InvalidQueryError, match="Unrecognized region string"): + Catalogs._create_adql_region(region="Badshape ICRS 202.4656816 +47.1999842 0.04") + + # Invalid list of coord pairs + with pytest.raises(InvalidQueryError, match="Invalid iterable region format"): + Catalogs._create_adql_region( + region=[57.376, 24.053, 56.391, 24.622, 56.025, 24.049, 56.616, 24.291] + ) + + if HAS_REGIONS: + # Invalid astropy region + with pytest.raises(TypeError, match="Unsupported region type"): + Catalogs._create_adql_region( + region=CirclePixelRegion(PixCoord(x=42, y=43), 4.2) + ) + + +def test_catalogs_parse_numeric_expression(patch_tap): + # Handling between + predicate = Catalogs._parse_numeric_expr("dec", "5..10") + assert predicate == "dec BETWEEN 5 AND 10" + + # Handling inequalities + predicate = Catalogs._parse_numeric_expr("teff", "<1") + assert predicate == "teff < 1" + + # Handling specific values + predicate = Catalogs._parse_numeric_expr("gaiabp", "1") + assert predicate == "gaiabp = 1.0" + + # Passing not numeric str + with pytest.raises(InvalidQueryError, match="is numeric; unsupported value"): + Catalogs._parse_numeric_expr("dec", "notnumeric") + + +def test_catalogs_parse_temporal_expression(patch_tap): + # Handling between + predicate = Catalogs._parse_temporal_expr("time", "2024-01-01..2024-12-31") + assert predicate == "time BETWEEN '2024-01-01 00:00:00' AND '2024-12-31 00:00:00'" + + # Handling inequalities + predicate = Catalogs._parse_temporal_expr("datetime", ">=2020-06-01") + assert predicate == "datetime >= '2020-06-01 00:00:00'" + + # Handling specific values + predicate = Catalogs._parse_temporal_expr("obs_time", "2025-04-01 12:00:00") + assert predicate == "obs_time BETWEEN '2025-04-01 12:00:00' AND '2025-04-01 12:00:01'" + + # Passing not datetime str + predicate = Catalogs._parse_temporal_expr("dec", "notdatetime") + assert predicate == "dec = 'notdatetime'" + + # Handling microseconds + predicate = Catalogs._parse_temporal_expr("datetime", ">=2020-06-01 10:00:00.0001") + assert predicate == "datetime >= '2020-06-01 10:00:00'" + + # Handling year-only str + predicate = Catalogs._parse_temporal_expr("time", "2025") + assert predicate == "time = '2025'" + + # Handling date str + predicate = Catalogs._parse_temporal_expr("time", "2020-08-01") + assert predicate == "time BETWEEN '2020-08-01 00:00:00' AND '2020-08-01 00:00:01'" + + # Handling astropy Time + predicate = Catalogs._parse_temporal_expr("obs_time", Time('2000-01-01 12:30:00')) + assert predicate == "obs_time BETWEEN '2000-01-01 12:30:00' AND '2000-01-01 12:30:01'" + + # Handling datetime + predicate = Catalogs._parse_temporal_expr("obs_time", datetime(2024, 6, 1, 8, 0, 1)) + assert predicate == "obs_time BETWEEN '2024-06-01 08:00:01' AND '2024-06-01 08:00:02'" + + +def test_catalogs_format_scalar_predicate(patch_tap): + # Handling bool + predicate = Catalogs._format_scalar_predicate( + "var", True + ) + assert predicate == 'var = 1' + + # Handling num + predicate = Catalogs._format_scalar_predicate( + "e_lum", 1, is_numeric=True + ) + assert predicate == 'e_lum = 1' + + # Handling str(num) + predicate = Catalogs._format_scalar_predicate( + "e_lum", "1", is_numeric=True + ) + assert predicate == 'e_lum = 1.0' - with pytest.warns(InputWarning) as i_w: + # Handling ! nots + predicate = Catalogs._format_scalar_predicate( + "e_lum", "!1", is_numeric=True + ) + assert predicate == 'NOT (e_lum = 1.0)' + + # Handling wildcard * + predicate = Catalogs._format_scalar_predicate( + "var", "WILDCARD*" + ) + assert predicate == "var LIKE 'WILDCARD%'" + + # Handling wildcard * and nots + predicate = Catalogs._format_scalar_predicate( + "var", "!WILDCARD*" + ) + assert predicate == "NOT (var LIKE 'WILDCARD%')" + + # Handling wildcard % + predicate = Catalogs._format_scalar_predicate( + "var", "WILDCARD%" + ) + assert predicate == "var LIKE 'WILDCARD%'" + + # Handling wildcard % and nots + predicate = Catalogs._format_scalar_predicate( + "var", "!WILDCARD%" + ) + assert predicate == "NOT (var LIKE 'WILDCARD%')" + + # Handling temporal + predicate = Catalogs._format_scalar_predicate( + "time", "!1", is_temporal=True + ) + assert predicate == "NOT (time = '1')" + + +def test_catalogs_combine_predicates(patch_tap): + # No predicates + result = Catalogs._combine_predicates([], []) + assert result == "" + + # One positive predicate + result = Catalogs._combine_predicates(["ra > 5"], []) + assert result == "ra > 5" + + # Multiple positive predicate + result = Catalogs._combine_predicates( + ["ra > 5", "dec < 0"], + [] + ) + assert result == "(ra > 5 OR dec < 0)" + + # Multiple negative predicates + result = Catalogs._combine_predicates( + [], + ["ra != 5", "dec != 0"] + ) + assert result == "ra != 5 AND dec != 0" + + # Multiple positive and negative predicates + result = Catalogs._combine_predicates( + ["ra > 5", "dec < 0"], + ["ra != 10"] + ) + assert result == "(ra != 10) AND (ra > 5 OR dec < 0)" + + +def test_catalogs_build_numeric_list_predicate(patch_tap): + # Multiple positive nums + result = Catalogs._build_numeric_list_predicate( + "ra", + pos_items=[1, 2, 3], + neg_items=[] + ) + assert result == "ra IN (1, 2, 3)" + + # Multiple positive bools + result = Catalogs._build_numeric_list_predicate( + "tessflag", + pos_items=[True, False], + neg_items=[] + ) + assert result == "tessflag IN (1, 0)" + + # Multiple positive inequalities and ranges + result = Catalogs._build_numeric_list_predicate( + "dec", + pos_items=["<5", "10..20"], + neg_items=[] + ) + assert "< 5" in result + assert "BETWEEN 10 AND 20" in result + + # Single positive num + result = Catalogs._build_numeric_list_predicate( + "ra", + pos_items=[np.int64(7)], + neg_items=[] + ) + assert "ra IN (7.0)" in result + + # Positive and negative nums + result = Catalogs._build_numeric_list_predicate( + "ra", + pos_items=[1, 2], + neg_items=[">5"] + ) + assert "ra IN (1, 2)" in result + assert "ra > 5" in result + assert "AND" in result + + # Unsupported numeric value type + with pytest.raises(InvalidQueryError, match="Unsupported numeric value type"): + Catalogs._build_numeric_list_predicate( + "ra", + pos_items=[{"not": "a number"}], + neg_items=[] + ) + + +def test_catalogs_build_string_list_predicate(patch_tap): + # Multiple positive strs + result = Catalogs._build_string_list_predicate( + "var", + pos_items=["str", "str"], + neg_items=[] + ) + assert result == "var IN ('str', 'str')" + + # Multiple positive bools + result = Catalogs._build_string_list_predicate( + "var", + pos_items=[True, False], + neg_items=[] + ) + assert result == "var IN (1, 0)" + + # Multiple positive strs with wildcard + result = Catalogs._build_string_list_predicate( + "var", + pos_items=["WILDCARD%", "str"], + neg_items=[] + ) + assert "var IN ('str')" in result + assert "LIKE 'WILDCARD%" in result + assert "OR" in result + + # Multiple positive nums + result = Catalogs._build_string_list_predicate( + "var", + pos_items=[1, 0], + neg_items=[] + ) + assert result == "var IN (1, 0)" + + +def test_catalogs_build_temporal_list_predicate(patch_tap): + # Multiple positive temps + result = Catalogs._build_temporal_list_predicate( + "var", + pos_items=["<2020-06-01", ">2025-06-01"], + neg_items=[] + ) + assert result == "(var < '2020-06-01 00:00:00' OR var > '2025-06-01 00:00:00')" + + # Multiple positive inequalities and ranges + result = Catalogs._build_temporal_list_predicate( + "var", + pos_items=[">2025-06-01", "2020-06-01..2020-12-31"], + neg_items=[] + ) + assert "var > '2025-06-01 00:00:00'" in result + assert "BETWEEN '2020-06-01 00:00:00' AND '2020-12-31 00:00:00'" in result + + +def test_catalogs_format_criteria_conditions(patch_tap): + # Multiple numeric cols and singular criteria + criteria = {"ra": 5, "dec": 10} + result = Catalogs._format_criteria_conditions( + CatalogCollection("tic"), + "dbo.catalogrecord", + criteria + ) + assert result == ["ra = 5", "dec = 10"] + + # Str cols and singular criteria + criteria = {"obj_type": "STAR"} + result = Catalogs._format_criteria_conditions( + CatalogCollection("tic"), + "dbo.catalogrecord", + criteria + ) + assert result == ["obj_type = 'STAR'"] + + # Multiple cols and multiple criteria + criteria = {"ra": [1, 2, 3], "dec": [">5", "<10"]} + result = Catalogs._format_criteria_conditions( + CatalogCollection("tic"), + "dbo.catalogrecord", + criteria + ) + assert any("ra IN" in r or "ra >" in r for r in result) + assert any("dec <" in r or "dec >" in r for r in result) + + # Str cols and multiple criteria + criteria = {"obj_type": ["STAR", "!GALAXY"]} + result = Catalogs._format_criteria_conditions( + CatalogCollection("tic"), + "dbo.catalogrecord", + criteria + ) + assert any("NOT" in r for r in result) + assert any("STAR" in r for r in result) + + # Empty criteria + criteria = {"ra": []} + result = Catalogs._format_criteria_conditions( + CatalogCollection("tic"), + "dbo.catalogrecord", + criteria + ) + assert result == ["1=0"] + + +def test_catalogs_invalid_tap_query(patch_tap): + # This will trigger a DALQueryError in the mock TAP service + # when 'invalid' is found in the query string + with pytest.raises(InvalidQueryError, match="Simulated TAP query error"): + Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + radius=0.002 * u.deg, + allwise='invalid' + ) + + with pytest.raises(InvalidQueryError, match="Simulated TAP query error"): + Catalogs.query_region( + regionCoords, + radius=0.002 * u.deg, + collection="tic", + allwise='invalid' + ) + + with pytest.raises(InvalidQueryError, match="Simulated TAP query error"): + Catalogs.query_object( + "M10", + radius=.001, + collection="TIC", + allwise='invalid' + ) + + +def test_catalogs_invalid_spatial_query(patch_tap): + # Force spatial query to fail + patch_tap.search = MagicMock(side_effect=DALQueryError("spatial failed")) + with pytest.raises(InvalidQueryError, match="does not support spatial queries"): + Catalogs.query_criteria( + collection="tic", + coordinates=regionCoords, + radius=0.002 * u.deg, + ) + + +def test_catalogs_query_hsc_matchid_async(patch_post): + with pytest.warns(AstropyDeprecationWarning, match="This function is deprecated"): + responses = Catalogs.query_hsc_matchid_async(82371983) + assert isinstance(responses, list) + + with pytest.warns(AstropyDeprecationWarning, match="This function is deprecated"): + responses = Catalogs.query_hsc_matchid_async(82371983, version=2) + assert isinstance(responses, list) + + with pytest.warns((AstropyDeprecationWarning, InputWarning)) as record: Catalogs.query_hsc_matchid_async(82371983, version=5) - assert "Invalid HSC version number" in str(i_w[0].message) + messages = [str(w.message) for w in record] + assert any("This function is deprecated" in m for m in messages) + assert any("Invalid HSC version number" in m for m in messages) -def test_catalogs_query_hsc_matchid(): - result = Catalogs.query_hsc_matchid(82371983) - assert isinstance(result, Table) +def test_catalogs_query_hsc_matchid(patch_post): + with pytest.warns(AstropyDeprecationWarning, match="This function is deprecated"): + result = Catalogs.query_hsc_matchid(82371983) + assert isinstance(result, Table) -def test_catalogs_get_hsc_spectra_async(): - responses = Catalogs.get_hsc_spectra_async() - assert isinstance(responses, list) +def test_catalogs_get_hsc_spectra_async(patch_post): + with pytest.warns(AstropyDeprecationWarning, match="This function is deprecated"): + responses = Catalogs.get_hsc_spectra_async() + assert isinstance(responses, list) -def test_catalogs_get_hsc_spectra(): - result = Catalogs.get_hsc_spectra() - assert isinstance(result, Table) +def test_catalogs_get_hsc_spectra(patch_post): + with pytest.warns(AstropyDeprecationWarning, match="This function is deprecated"): + result = Catalogs.get_hsc_spectra() + assert isinstance(result, Table) -def test_catalogs_download_hsc_spectra(tmpdir): - allSpectra = Catalogs.get_hsc_spectra() +def test_catalogs_download_hsc_spectra(patch_post, tmpdir): + with pytest.warns(AstropyDeprecationWarning, match="This function is deprecated"): + allSpectra = Catalogs.get_hsc_spectra() - # actually download the products - result = Catalogs.download_hsc_spectra(allSpectra[10], download_dir=str(tmpdir)) - assert isinstance(result, Table) + # Actually download the products + result = Catalogs.download_hsc_spectra(allSpectra[10], download_dir=str(tmpdir)) + assert isinstance(result, Table) + + # Just get the curl script + result = Catalogs.download_hsc_spectra(allSpectra[20:24], + download_dir=str(tmpdir), curl_flag=True) + assert isinstance(result, Table) + + +########################### +# CatalogCollection tests # +########################### + + +def test_catalog_collection_discover_collections(patch_tap): + collections = CatalogCollection.discover_collections() + assert isinstance(collections, Table) + assert len(collections) > 0 + assert "collection_name" in collections.colnames + assert "parent_collection" in collections.colnames + + +def test_catalog_collection_get_parent_collection(patch_tap): + parent = CatalogCollection.get_parent_collection("tic") + assert parent == "tic" + + # Error if collection not a string + with pytest.raises(InvalidQueryError): + CatalogCollection.get_parent_collection(123) + + # Error if collection not found + with pytest.raises(InvalidQueryError, match="Collection 'fake' not found"): + CatalogCollection.get_parent_collection("fake") + + +def test_catalog_collection_tap_get_catalog_metadata(patch_tap): + cc = CatalogCollection("tic") + default_catalog = cc.get_default_catalog() + default_metadata = cc.get_catalog_metadata(default_catalog) + assert isinstance(default_metadata, CatalogMetadata) + assert isinstance(default_metadata.column_metadata, Table) + assert isinstance(default_metadata.ra_column, str) + assert isinstance(default_metadata.dec_column, str) + assert isinstance(default_metadata.supports_spatial_queries, bool) + + assert len(default_metadata.column_metadata) > 0 + assert default_metadata.ra_column in default_metadata.column_metadata["column_name"] + assert default_metadata.dec_column in default_metadata.column_metadata["column_name"] + + metadata_cached = cc.get_catalog_metadata(default_catalog) + assert default_metadata is metadata_cached + + +def test_catalog_collection_get_default_catalog(patch_tap): + cc = CatalogCollection("tic") + catalogs = cc._fetch_catalogs() + default = cc.get_default_catalog() + + assert isinstance(catalogs, Table) + assert len(catalogs) > 0 + assert catalogs.colnames == ['catalog_name', 'description'] + + assert not default.startswith("tap_schema") + assert default.casefold() in [name.casefold() for name in catalogs["catalog_name"]] + assert DEFAULT_CATALOGS["tic"] == default + + # First non-tap_schema + cc = CatalogCollection("tic") + cc.name = "fake" + fake_catalogs = Table({ + "catalog_name": [ + "tap_schema.tables", + "tap_schema.columns", + "real_catalog", + "real_catalog_2", + ], + "description": ["", "", "", ""], + }) + cc._fetch_catalogs = MagicMock(return_value=fake_catalogs) + assert isinstance(catalogs, Table) + assert len(catalogs) > 0 + + default = cc.get_default_catalog() + assert default == "real_catalog" + + # All are tap_schema + cc = CatalogCollection("tic") + cc.name = "fake" + fake_catalogs = Table({ + "catalog_name": [ + "tap_schema.tables", + "tap_schema.columns", + ], + "description": ["", ""], + }) + cc._fetch_catalogs = MagicMock(return_value=fake_catalogs) + assert isinstance(catalogs, Table) + assert len(catalogs) > 0 + + default = cc.get_default_catalog() + assert default == "tap_schema.tables" + + +def test_catalog_collection_run_tap_query(patch_tap): + cc = CatalogCollection("tic") + + adql_str = ( + "SELECT TOP 10 solution_id, designation, source_id, ra, dec " + "FROM gaia_source WHERE " + "ra BETWEEN 10 AND 11 AND dec BETWEEN 12 AND 13" + ) + result = cc.run_tap_query(adql_str) - # just get the curl script - result = Catalogs.download_hsc_spectra(allSpectra[20:24], - download_dir=str(tmpdir), curl_flag=True) assert isinstance(result, Table) + assert len(result) > 0 + + query = get_patch_tap_query(patch_tap) + assert adql_str in query + + +def test_catalog_collection_invalid_run_tap_query(patch_tap): + cc = CatalogCollection("tic") + with pytest.raises(InvalidQueryError, match="TAP query failed for collection 'tic'"): + adql_str = "invalid" + cc.run_tap_query(adql_str) + + +def test_catalog_collection_grouped_fetch_catalogs(patch_tap): + name = "dbo" + cc = CatalogCollection(name) + _ = cc._fetch_catalogs() + query = get_patch_tap_query(patch_tap) + assert f"WHERE table_name LIKE '{name}" in query + + +def test_catalog_collection_verify_catalog(patch_tap): + cc = CatalogCollection("tic") + default_catalog = cc.get_default_catalog() + + # Valid catalog + assert isinstance(cc._verify_catalog(default_catalog), str) + assert cc._verify_catalog(default_catalog) == 'dbo.catalogrecord' + + +def test_catalog_collection_invalid_verify_catalog(patch_tap): + cc = CatalogCollection("tic") + + # Ambiguous + fake_catalogs = Table({ + "catalog_name": [ + "mission1.catalogA", + "mission2.catalogA", + ], + "description": ["", ""], + }) + cc._fetch_catalogs = MagicMock(return_value=fake_catalogs) + + with pytest.raises(InvalidQueryError, match="is ambiguous for collection"): + cc._verify_catalog("catalogA") + + # Invalid catalog + with pytest.raises(InvalidQueryError, match="Catalog 'fake' is not recognized for collection 'tic'"): + cc._verify_catalog("fake") + + +def test_catalog_collection_invalid_get_column_metadata(patch_tap): + cc = CatalogCollection("tic") + + empty_result = Table( + names=["column_name", "datatype", "unit", "ucd", "description"] + ) + cc.tap_service.run_sync = MagicMock(return_value=empty_result) + + with pytest.raises( + InvalidQueryError, + match="Catalog 'fake_catalog' not found in collection 'tic'" + ): + cc._get_column_metadata("fake_catalog") + + +def test_catalog_collection_verify_criteria(patch_tap): + cc = CatalogCollection("tic") + default_catalog = cc.get_default_catalog() + + # Valid filters + assert cc._verify_criteria(default_catalog) is None + assert cc._verify_criteria(default_catalog, gaiabp=1) is None + assert cc._verify_criteria(default_catalog, gaiabp=1, teff=1) is None + + +def test_catalog_collection_invalid_verify_criteria(patch_tap): + cc = CatalogCollection("tic") + default_catalog = cc.get_default_catalog() + + close_match_col = "gaiagaiabp" + with pytest.raises(InvalidQueryError, match=f"Filter '{close_match_col}' is not recognized for collection " + f"'{cc.name}' and catalog '{default_catalog}'. Did you mean 'gaiabp'?"): + cc._verify_criteria(default_catalog, gaiagaiabp=1) + + invalid_col = "fake_column" + with pytest.raises(InvalidQueryError, match=f"Filter '{invalid_col}' is not recognized for collection " + f"'{cc.name}' and catalog '{default_catalog}'."): + cc._verify_criteria(default_catalog, fake_column=1) + + +def test_catalog_collection_invalid_spatial_query(patch_tap): + cc = CatalogCollection("tic") + + # Force only spatial query to fail + patch_tap.search = MagicMock(side_effect=DALQueryError("spatial failed")) + default_catalog = cc.get_default_catalog() + metadata = cc.get_catalog_metadata(default_catalog) + + assert metadata.supports_spatial_queries is False + assert patch_tap.search.called + + +def test_catalog_collection_invalid_collection_type(patch_tap): + # Error if collection name is not a string + with pytest.raises(ValueError, match="Collection name must be a string, got "): + CatalogCollection(123) ###################### diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index da6e8fe3e7..f33dfc4fd2 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -1,24 +1,38 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +import json import logging +import os from pathlib import Path + +import astropy.units as u import numpy as np -import os import pytest -import json - -from requests.models import Response - -from astropy.table import Table, unique from astropy.coordinates import SkyCoord from astropy.io import fits -import astropy.units as u - -from astroquery.mast import Observations, utils, Mast, Catalogs, Hapcut, Tesscut, Zcut, MastMissions +from astropy.table import Table, unique +from astropy.time import Time +from requests.models import Response +from astroquery.mast import ( + Catalogs, + Hapcut, + Mast, + MastMissions, + Observations, + Tesscut, + Zcut, + utils, +) + +from ...exceptions import ( + InputWarning, + InvalidQueryError, + MaxResultsWarning, + NoResultsWarning, +) +from ..catalog_collection import DEFAULT_CATALOGS, CatalogCollection, CatalogMetadata from ..utils import ResolverError -from ...exceptions import (InputWarning, InvalidQueryError, MaxResultsWarning, - NoResultsWarning) @pytest.fixture(scope="module") @@ -1039,388 +1053,372 @@ def test_observations_get_cloud_uris_no_duplicates(self, msa_product_table, rese # CatalogClass tests # ###################### - # query functions - def test_catalogs_query_region_async(self): - in_rad = 0.001 * u.deg - responses = Catalogs.query_region_async("158.47924 -7.30962", - radius=in_rad, - catalog="Galex") - assert isinstance(responses, list) - - # Default catalog is HSC - responses = Catalogs.query_region_async("322.49324 12.16683", - radius=in_rad) - assert isinstance(responses, list) - - responses = Catalogs.query_region_async("322.49324 12.16683", - radius=in_rad, - catalog="panstarrs", - table="mean") - assert isinstance(responses, Response) + def test_catalogs_collection(self): + # Default collection should be HSC + c = Catalogs() + assert c.collection == "hsc" + assert c.catalog == "dbo.SumMagAper2CatView" + + # Initialize with a different collection + c = Catalogs(collection="gaiadr3") + assert c.collection == "gaiadr3" + assert c.catalog == "dbo.gaia_source" + + # Initialize with a different collection and catalog + c = Catalogs(collection="ullyses", catalog="publications") + assert c.collection == "ullyses" + assert c.catalog == "dbo.publications" + + # Set the collection + c.collection = "caom" + assert c.collection == "caom" + assert c.catalog == "dbo.obspointing" + + def test_catalogs_get_collections(self): + collections = Catalogs.get_collections() + assert isinstance(collections, Table) + assert "collection_name" in collections.colnames + + def test_catalogs_get_catalogs(self): + catalogs = Catalogs.get_catalogs() + assert isinstance(catalogs, Table) + assert "catalog_name" in catalogs.colnames + assert "description" in catalogs.colnames + + def test_catalogs_get_column_metadata(self): + metadata = Catalogs.get_column_metadata("hsc", "dbo.SumMagAper2CatView") + assert isinstance(metadata, Table) + assert len(metadata) > 0 + assert "column_name" in metadata.colnames + assert "datatype" in metadata.colnames + assert "unit" in metadata.colnames + assert "ucd" in metadata.colnames + assert "description" in metadata.colnames + + def test_catalogs_supports_spatial_queries(self): + assert Catalogs.supports_spatial_queries("hsc", "dbo.SumMagAper2CatView") + assert not Catalogs.supports_spatial_queries("ullyses", "dbo.publications") - def test_catalogs_query_region(self): - def check_result(result, row, exp_values): - assert isinstance(result, Table) - for k, v in exp_values.items(): - assert result[row][k] == v - - in_radius = 0.1 * u.deg - result = Catalogs.query_region("158.47924 -7.30962", - radius=in_radius, - catalog="Gaia") - row = np.where(result['source_id'] == '3774902350511581696') - check_result(result, row, {'solution_id': '1635721458409799680'}) - - result = Catalogs.query_region("322.49324 12.16683", - radius=0.001*u.deg, - catalog="HSC", - magtype=2) - row = np.where(result['MatchID'] == '8150896') - - with pytest.warns(MaxResultsWarning): - result = Catalogs.query_region("322.49324 12.16683", catalog="HSC", magtype=2, nr=5) - - check_result(result, row, {'NumImages': 14, 'TargetName': 'M15'}) - - result = Catalogs.query_region("322.49324 12.16683", - radius=0.001*u.deg, - catalog="HSC", - version=2, - magtype=2) - row = np.where(result['MatchID'] == '82361658') - check_result(result, row, {'NumImages': 11, 'TargetName': 'NGC7078'}) - - result = Catalogs.query_region("322.49324 12.16683", - radius=in_radius, - catalog="Gaia", - version=1) - row = np.where(result['source_id'] == '1745948323734098688') - check_result(result, row, {'solution_id': '1635378410781933568'}) - result = Catalogs.query_region("322.49324 12.16683", - radius=0.01*u.deg, - catalog="Gaia", - version=2) - - row = np.where(result['source_id'] == '1745947739618544000') - check_result(result, row, {'solution_id': '1635721458409799680'}) - - result = Catalogs.query_region("322.49324 12.16683", - radius=0.01*u.deg, catalog="panstarrs", - table="mean", - columns=['objName', 'objID', 'yFlags', 'distance']) - row = np.where((result['objName'] == 'PSO J322.4622+12.1920') & (result['yFlags'] == 16777496)) - assert isinstance(result, Table) - np.testing.assert_allclose(result[row]['distance'], 0.039381703406789904) - - result = Catalogs.query_region("158.47924 -7.30962", - radius=in_radius, - catalog="Galex") - in_radius_arcmin = 0.1*u.deg.to(u.arcmin) - distances = list(result['distance_arcmin']) + def test_catalogs_query_criteria(self): + # Positional query with multiple filters + c = Catalogs() + search_coord = SkyCoord(322.49324, 12.16683, unit="deg") + select_cols = ["matchid", "matchra", "matchdec", "numimages", "starttime", "targetname"] + result = c.query_criteria( + coordinates=search_coord, + radius="2 arcsec", + sort_by=["numimages", "starttime"], + sort_desc=[False, True], + targetname=["M-15", "NGC*"], + starttime=">2010", + limit=5, + select_cols=select_cols, + ) assert isinstance(result, Table) - assert max(distances) <= in_radius_arcmin - - result = Catalogs.query_region("158.47924 -7.30962", - radius=in_radius, - catalog="tic") - row = np.where(result['ID'] == '841736289') - second_id = result[1]['ID'] - check_result(result, row, {'gaiaqflag': 1}) - np.testing.assert_allclose(result[row]['RA_orig'], 158.475246786483) - - result = Catalogs.query_region("158.47924 -7.30962", - radius=in_radius, - catalog="tic", - pagesize=1, - page=2) + assert len(result) == 5 + assert all(c in result.colnames for c in select_cols) + assert all(val == "M-15" or str(val).startswith("NGC") for val in result["targetname"]) + assert all(result["starttime"] > Time("2010-01-01T00:00:00")) + # Assert that all results are within 2 arcsec of the specified coordinates + coords = SkyCoord(result["matchra"], result["matchdec"], unit="deg") + separation = coords.separation(search_coord) + assert all(separation <= 2 * u.arcsec) + # Assert that results are sorted by numimages ascending and then starttime descending + assert all(result["numimages"][i] <= result["numimages"][i + 1] for i in range(len(result) - 1)) + assert all( + result["starttime"][i] >= result["starttime"][i + 1] + for i in range(len(result) - 1) + if result["numimages"][i] == result["numimages"][i + 1] + ) + + # Non-positional query with multiple filters and select_cols + select_cols = [ + "target_name_ullyses", + "target_classification", + "known_binary", + "sp_class", + "gaia_parallax", + "star_teff", + "coordinate_epoch", + "spectral_type_ref", + ] + result = c.query_criteria( + collection="ullyses", + catalog="sciencemetadata", + target_name_ullyses="NGC*", + target_classification=["!Galaxy", "!Late O Dwarf"], + known_binary=False, + sp_class=["O", "B"], + gaia_parallax=["<-0.01", ">=0", "!<-0.3"], + star_teff="30000..50000", + coordinate_epoch=2016, + spectral_type_ref=[51, 18, 59], + select_cols=select_cols, + ) assert isinstance(result, Table) - assert len(result) == 1 - assert second_id == result[0]['ID'] - - result = Catalogs.query_region("158.47924 -7.30962", - radius=in_radius, - catalog="ctl") - row = np.where(result['ID'] == '56662064') - check_result(result, row, {'TYC': '4918-01335-1'}) - - result = Catalogs.query_region("210.80227 54.34895", - radius=1*u.deg, - catalog="diskdetective") - row = np.where(result['designation'] == 'J140544.95+535941.1') - check_result(result, row, {'ZooniverseID': 'AWI0000r57'}) - - def test_catalogs_query_object_async(self): - responses = Catalogs.query_object_async("M10", - radius=.02, - catalog="TIC") - assert isinstance(responses, list) - - def test_catalogs_query_object(self): - def check_result(result, exp_values): - assert isinstance(result, Table) - for k, v in exp_values.items(): - assert v in result[k] - - result = Catalogs.query_object("M10", - radius=.001, - catalog="TIC") - check_result(result, {'ID': '1305764225'}) - second_id = result[1]['ID'] - - result = Catalogs.query_object("M10", - radius=.001, - catalog="TIC", - pagesize=1, - page=2) - assert isinstance(result, Table) - assert len(result) == 1 - assert second_id == result[0]['ID'] - - result = Catalogs.query_object("M10", - radius=.001, - catalog="HSC", - magtype=1) - check_result(result, {'MatchID': '667727'}) - - result = Catalogs.query_object("M10", - radius=.001, - catalog="panstarrs", - table="mean", - columns=['objName', 'objID']) - check_result(result, {'objName': 'PSO J254.2873-04.1006'}) - - result = Catalogs.query_object("M10", - radius=0.18, - catalog="diskdetective") - check_result(result, {'designation': 'J165749.79-040315.1'}) - - result = Catalogs.query_object("M10", - radius=0.001, - catalog="Gaia", - version=1) - distances = list(result['distance']) - radius_arcmin = 0.01 * u.deg.to(u.arcmin) + assert len(result) > 0 + assert all(str(val).startswith("NGC") for val in result["target_name_ullyses"]) + assert all(result["target_classification"] != "Galaxy") + assert all(result["target_classification"] != "Late O Dwarf") + assert not any(result["known_binary"]) + assert all(np.isin(result["sp_class"], ["O", "B"])) + assert all( + ((result["gaia_parallax"] < -0.01) | (result["gaia_parallax"] >= 0)) + & ~(result["gaia_parallax"] < -0.3) + ) + assert all((result["star_teff"] >= 30000) & (result["star_teff"] <= 50000)) + assert all(result["coordinate_epoch"] == 2016) + assert all(np.isin(result["spectral_type_ref"], [51, 18, 59])) + assert all(c in result.colnames for c in select_cols) + + # Test offset + result = c.query_criteria(collection="hsc", numimages=8, sort_by="matchid", limit=5) + result_offset = c.query_criteria(collection="hsc", numimages=8, sort_by="matchid", limit=5, offset=2) + assert result_offset["matchid"][0] == result["matchid"][2] + + # Count only + result_count = c.query_criteria(collection="goods", class_star=0.23, count_only=True) + assert isinstance(result_count, (int, np.integer)) + assert result_count > 0 + + # Temporal filters and filter passed in through filters argument + result = c.query_criteria(collection='caom', + catalog='caommembers', + limit=10, + recordcreated=['>2014-01-01T00:00:00', '<2000'], + recordmodified='2021-03-01..2021-03-31', + filters={'collection': 'GALEX'}) assert isinstance(result, Table) - assert max(distances) < radius_arcmin - - result = Catalogs.query_object("TIC 441662144", - radius=0.001, - catalog="ctl") - check_result(result, {'ID': '441662144'}) - - result = Catalogs.query_object('M10', - radius=0.08, - catalog='plato') - assert 'PICidDR1' in result.colnames - - def test_catalogs_query_criteria_async(self): - # without position - responses = Catalogs.query_criteria_async(catalog="Tic", - Bmag=[30, 50], - objType="STAR") - assert isinstance(responses, list) - - responses = Catalogs.query_criteria_async(catalog="ctl", - Bmag=[30, 50], - objType="STAR") - assert isinstance(responses, list) - - responses = Catalogs.query_criteria_async(catalog="DiskDetective", - state=["inactive", "disabled"], - oval=[8, 10], - multi=[3, 7]) - assert isinstance(responses, list) - - # with position - responses = Catalogs.query_criteria_async(catalog="Tic", - object_name="M10", - objType="EXTENDED") - assert isinstance(responses, list) - - responses = Catalogs.query_criteria_async(catalog="CTL", - object_name="M10", - objType="EXTENDED") - assert isinstance(responses, list) - - responses = Catalogs.query_criteria_async(catalog="DiskDetective", - object_name="M10", - radius=2, - state="complete") - assert isinstance(responses, list) + assert len(result) <= 10 + assert all(c in result.colnames for c in ['recordcreated', 'collection']) + assert all( + (result['recordcreated'] > Time('2014-01-01T00:00:00')) + | (result['recordcreated'] < Time('2000-01-01T00:00:00')) + ) + assert all((result['recordmodified'] >= Time('2021-03-01T00:00:00')) + & (result['recordmodified'] <= Time('2021-03-31T23:59:59'))) + assert all(result['collection'] == 'GALEX') + + def test_catalogs_query_criteria_error(self): + # No results should warn user + # with pytest.warns(NoResultsWarning): + # Catalogs.query_criteria(collection="ps1_dr2", skycellid=-1) - responses = Catalogs.query_criteria_async(catalog="panstarrs", - table="mean", - object_name="M10", - radius=.02, - qualityFlag=48) - assert isinstance(responses, Response) - - def test_catalogs_query_criteria(self): - def check_result(result, exp_vals): - assert isinstance(result, Table) - for k, v in exp_vals.items(): - assert v in result[k] + with pytest.warns(NoResultsWarning): + Catalogs.query_criteria(collection="classy", target=[]) - # without position - result = Catalogs.query_criteria(catalog="Tic", - Bmag=[30, 50], - objType="STAR") - check_result(result, {'ID': '81609218'}) - second_id = result[1]['ID'] - - result = Catalogs.query_criteria(catalog="Tic", - Bmag=[30, 50], - objType="STAR", - pagesize=1, - page=2) + def test_catalogs_query_region(self): + # Region search with polygon + select_cols = ["target_name", "obs_id", "s_ra", "s_dec", "s_region"] + result = Catalogs.query_region( + collection="caom", + region="POLYGON ICRS 18.85 -6.95 18.86 -6.95 18.86 -6.94 18.85 -6.94", + limit=5, + select_cols=select_cols, + ) assert isinstance(result, Table) - assert len(result) == 1 - assert second_id == result[0]['ID'] - - result = Catalogs.query_criteria(catalog="ctl", - Tmag=[10.5, 11], - POSflag="2mass") - check_result(result, {'ID': '291067184'}) - - result = Catalogs.query_criteria(catalog="DiskDetective", - state=["inactive", "disabled"], - oval=[8, 10], - multi=[3, 7]) - check_result(result, {'designation': 'J003920.04-300132.4'}) - - # with position - result = Catalogs.query_criteria(catalog="Tic", - object_name="M10", objType="EXTENDED") - check_result(result, {'ID': '10000732589'}) - - result = Catalogs.query_criteria(object_name='TIC 291067184', - catalog="ctl", - Tmag=[10.5, 11], - POSflag="2mass") - check_result(result, {'Tmag': 10.893}) - - result = Catalogs.query_criteria(catalog="DiskDetective", - object_name="M10", - radius=2, - state="complete") - check_result(result, {'designation': 'J165628.40-054630.8'}) - - result = Catalogs.query_criteria(catalog="panstarrs", - object_name="M10", - radius=.01, - qualityFlag=32, - zoneID=10306, - columns=['objName', 'objID']) - check_result(result, {'objName': 'PSO J254.2861-04.1091'}) - - result = Catalogs.query_criteria(coordinates="158.47924 -7.30962", - radius=0.01, - catalog="PANSTARRS", - table="mean", - data_release="dr2", - nStackDetections=[("gte", "1")], - columns=["objName", "distance"], - sort_by=[("asc", "distance")]) + assert len(result) <= 5 + assert all(c in result.colnames for c in select_cols) + # Assert that all results are within a radius of the specified polygon + # We can't just check that the coordinates are within the polygon because + # they may intersect the polygon without the center being within it + coords = SkyCoord(result["s_ra"], result["s_dec"], unit="deg") + polygon = SkyCoord(18.855, -6.945, unit="deg") + separation = coords.separation(polygon) + assert all(separation <= 0.1 * u.deg) + + # Region search with circle + result = Catalogs.query_region( + collection="caom", region="CIRCLE ICRS 18.85 -6.95 0.01", limit=5, select_cols=select_cols + ) assert isinstance(result, Table) - assert result['distance'][0] <= result['distance'][1] - - # with case-insensitive keyword arguments - result = Catalogs.query_criteria(catalog="Tic", - bMAG=[30, 50], - objtype="STAR") - check_result(result, {'ID': '81609218'}) - - result = Catalogs.query_criteria(catalog="DiskDetective", - STATE=["inactive", "disabled"], - oVaL=[8, 10], - Multi=[3, 7]) - check_result(result, {'designation': 'J003920.04-300132.4'}) - - def test_catalogs_query_criteria_invalid_keyword(self): - # attempt to make a criteria query with invalid keyword - with pytest.raises(InvalidQueryError) as err_no_alt: - Catalogs.query_criteria(catalog='tic', not_a_keyword='TESS') - assert "Filter 'not_a_keyword' does not exist." in str(err_no_alt.value) - - # keyword is close enough for difflib to offer alternative - with pytest.raises(InvalidQueryError) as err_with_alt: - Catalogs.query_criteria(catalog='ctl', objectType="STAR") - assert 'objType' in str(err_with_alt.value) - - # region query with invalid keyword - with pytest.raises(InvalidQueryError) as err_region: - Catalogs.query_region('322.49324 12.16683', - radius=0.001*u.deg, - catalog='HSC', - invalid=2) - assert "Filter 'invalid' does not exist for catalog HSC." in str(err_region.value) - - # panstarrs criteria query with invalid keyword - with pytest.raises(InvalidQueryError) as err_ps_criteria: - Catalogs.query_criteria(coordinates="158.47924 -7.30962", - catalog="PANSTARRS", - table="mean", - data_release="dr2", - columns=["objName", "distance"], - sort_by=[("asc", "distance")], - obj_name='invalid') - assert 'objName' in str(err_ps_criteria.value) + assert len(result) <= 5 + assert all(c in result.colnames for c in select_cols) + # Assert that all results are within a radius of the specified circle + coords = SkyCoord(result["s_ra"], result["s_dec"], unit="deg") + center = SkyCoord(18.85, -6.95, unit="deg") + separation = coords.separation(center) + assert all(separation <= 0.1 * u.deg) + def test_catalogs_query_object(self): + # Object search + select_cols = ["target_name", "obs_id", "s_ra", "s_dec"] + result = Catalogs.query_object(collection="caom", object_name="M2", limit=5, select_cols=select_cols) + assert isinstance(result, Table) + assert len(result) <= 5 + assert all(c in result.colnames for c in select_cols) + # Assert that all results are within a radius of the specified object + coords = SkyCoord(result["s_ra"], result["s_dec"], unit="deg") + m2_coords = SkyCoord(323.36258, -0.82325, unit="deg") + separation = coords.separation(m2_coords) + assert all(separation <= 0.1 * u.deg) + + @pytest.mark.filterwarnings("ignore::astropy.utils.exceptions.AstropyDeprecationWarning") def test_catalogs_query_hsc_matchid_async(self): - catalogData = Catalogs.query_object("M10", - radius=.001, - catalog="HSC", - magtype=1) + catalogData = Catalogs.query_object("M10", radius=0.001, collection="HSC") responses = Catalogs.query_hsc_matchid_async(catalogData[0]) assert isinstance(responses, list) - responses = Catalogs.query_hsc_matchid_async(catalogData[0]["MatchID"]) + responses = Catalogs.query_hsc_matchid_async(catalogData[0]["matchid"]) assert isinstance(responses, list) + @pytest.mark.filterwarnings("ignore::astropy.utils.exceptions.AstropyDeprecationWarning") def test_catalogs_query_hsc_matchid(self): - catalogData = Catalogs.query_object("M10", - radius=.001, - catalog="HSC", - magtype=1) - matchid = catalogData[0]["MatchID"] + catalogData = Catalogs.query_object("M10", radius=0.001, collection="HSC") + matchid = str(catalogData[0]["matchid"]) result = Catalogs.query_hsc_matchid(catalogData[0]) assert isinstance(result, Table) - assert (result['MatchID'] == matchid).all() + assert (result["MatchID"].value == matchid).all() result2 = Catalogs.query_hsc_matchid(matchid) assert isinstance(result2, Table) assert len(result2) == len(result) - assert (result2['MatchID'] == matchid).all() + assert (result2["MatchID"] == matchid).all() + @pytest.mark.filterwarnings("ignore::astropy.utils.exceptions.AstropyDeprecationWarning") def test_catalogs_get_hsc_spectra_async(self): responses = Catalogs.get_hsc_spectra_async() assert isinstance(responses, list) + @pytest.mark.filterwarnings("ignore::astropy.utils.exceptions.AstropyDeprecationWarning") def test_catalogs_get_hsc_spectra(self): result = Catalogs.get_hsc_spectra() assert isinstance(result, Table) - assert result[np.where(result['MatchID'] == '19657846')] - assert result[np.where(result['DatasetName'] == 'HAG_J072657.06+691415.5_J8HPAXAEQ_V01.SPEC1D')] + assert result[np.where(result["MatchID"] == "19657846")] + assert result[np.where(result["DatasetName"] == "HAG_J072657.06+691415.5_J8HPAXAEQ_V01.SPEC1D")] + @pytest.mark.filterwarnings("ignore::astropy.utils.exceptions.AstropyDeprecationWarning") def test_catalogs_download_hsc_spectra(self, tmpdir): allSpectra = Catalogs.get_hsc_spectra() # actually download the products - result = Catalogs.download_hsc_spectra(allSpectra[10], - download_dir=str(tmpdir)) + result = Catalogs.download_hsc_spectra(allSpectra[10], download_dir=str(tmpdir)) assert isinstance(result, Table) for row in result: - if row['Status'] == 'COMPLETE': - assert os.path.isfile(row['Local Path']) + if row["Status"] == "COMPLETE": + assert os.path.isfile(row["Local Path"]) # just get the curl script - result = Catalogs.download_hsc_spectra(allSpectra[20:24], - download_dir=str(tmpdir), curl_flag=True) + result = Catalogs.download_hsc_spectra(allSpectra[20:24], download_dir=str(tmpdir), curl_flag=True) + assert isinstance(result, Table) + assert os.path.isfile(result["Local Path"][0]) + + ########################### + # CatalogCollection tests # + ########################### + + def test_catalog_collection_discover_collections(self): + collections = CatalogCollection.discover_collections() + assert isinstance(collections, Table) + assert len(collections) > 0 + assert "collection_name" in collections.colnames + assert "parent_collection" in collections.colnames + assert CatalogCollection._discovered_collections is not None + + def test_catalog_collection_get_parent_collection(self): + parent = CatalogCollection.get_parent_collection("TIC") + assert parent == "tic" + + # parent = CatalogCollection.get_parent_collection("tic_v82") + # assert parent == "mast_catalogs" + + @pytest.mark.parametrize("collection", ["caom"]) + # @pytest.mark.parametrize("collection", ["caom", "tic_v82"]) + def test_catalog_collection_get_catalogs(self, collection): + cc = CatalogCollection(collection) + catalogs = cc._fetch_catalogs() + assert isinstance(catalogs, Table) + assert len(catalogs) > 0 + assert catalogs.colnames == ["catalog_name", "description"] + + @pytest.mark.parametrize("collection", ["caom", "ullyses", "tic"]) + def test_catalog_collection_get_catalog_metadata(self, collection): + cc = CatalogCollection(collection) + default_catalog = cc.get_default_catalog() + default_metadata = cc.get_catalog_metadata(default_catalog) + assert isinstance(default_metadata, CatalogMetadata) + assert isinstance(default_metadata.column_metadata, Table) + assert isinstance(default_metadata.ra_column, str) + assert isinstance(default_metadata.dec_column, str) + assert isinstance(default_metadata.supports_spatial_queries, bool) + + assert len(default_metadata.column_metadata) > 1 + assert default_metadata.ra_column in default_metadata.column_metadata["column_name"] + assert default_metadata.dec_column in default_metadata.column_metadata["column_name"] + if collection != "ullyses": + assert default_metadata.supports_spatial_queries + + metadata_cache = cc._catalog_metadata_cache + assert default_catalog.casefold() in metadata_cache + default_metadata_cached = cc.get_catalog_metadata(default_catalog) + assert default_metadata is default_metadata_cached + + def test_catalog_collection_invalid_get_catalog_metadata(self): + cc = CatalogCollection("TIC") + invalid_catalog = "invalid_catalog" + with pytest.raises( + InvalidQueryError, match=f"Catalog '{invalid_catalog}' is not recognized for collection '{cc.name}'." + ): + cc.get_catalog_metadata(invalid_catalog) + + @pytest.mark.parametrize("collection", ["tic", "classy", "ullyses"]) + def test_catalog_collection_get_default_catalog(self, collection): + cc = CatalogCollection(collection) + catalogs = cc._fetch_catalogs() + default = cc.get_default_catalog() + + assert len(catalogs) > 1 + assert isinstance(catalogs, Table) + assert catalogs.colnames == ["catalog_name", "description"] + + assert not default.startswith("tap_schema") + assert default.casefold() in [name.casefold() for name in catalogs["catalog_name"]] + assert DEFAULT_CATALOGS[collection] == default + + def test_catalog_collection_run_tap_query(self): + cc = CatalogCollection("GAIADR3") + adql_str = ( + "SELECT TOP 10 solution_id, designation, source_id, ra, dec FROM gaia_source WHERE " + "ra BETWEEN 10 AND 11 AND dec BETWEEN 12 AND 13" + ) + result = cc.run_tap_query(adql_str) + assert isinstance(result, Table) - assert os.path.isfile(result['Local Path'][0]) + assert result.colnames == ["solution_id", "designation", "source_id", "ra", "dec"] + assert ((result["ra"] >= 10) & (result["ra"] <= 11)).all() + assert ((result["dec"] >= 12) & (result["dec"] <= 13)).all() + + def test_catalog_collection_verify_criteria(self): + cc = CatalogCollection("TIC") + default_catalog = cc.get_default_catalog() + + result = cc._verify_criteria(default_catalog) + assert result is None + + result = cc._verify_criteria(default_catalog, gaiabp=1) + assert result is None + + result = cc._verify_criteria(default_catalog, gaiabp=1, teff=1, e_gaiabp=1) + assert result is None + + close_match_col = "gaiagaiabp" + with pytest.raises( + InvalidQueryError, + match=f"Filter '{close_match_col}' is not recognized for collection " + f"'{cc.name}' and catalog '{default_catalog}'. Did you mean 'gaiabp'?", + ): + cc._verify_criteria(default_catalog, gaiagaiabp=1) + + invalid_col = "fake_column" + with pytest.raises( + InvalidQueryError, + match=f"Filter '{invalid_col}' is not recognized for collection " + f"'{cc.name}' and catalog '{default_catalog}'.", + ): + cc._verify_criteria(default_catalog, fake_column=1) ###################### # TesscutClass tests # diff --git a/docs/mast/mast_catalog.rst b/docs/mast/mast_catalog.rst index fe8c05d857..e53b16e8bd 100644 --- a/docs/mast/mast_catalog.rst +++ b/docs/mast/mast_catalog.rst @@ -1,332 +1,415 @@ - *************** Catalog Queries *************** -The Catalogs class provides access to a subset of the astronomical catalogs stored at MAST. -The catalogs currently available through this interface are: +The `~astroquery.mast.CatalogsClass` interface provides tools for discovering and querying the wide +range of astronomical catalogs hosted by MAST. These catalogs span multiple missions and surveys +and are organized into **collections**, each of which may contain one or more **catalogs** with +distinct schemas and capabilities. This interface is designed for **flexible, SQL-like querying** of +catalog data, including spatial searches and column-based filtering. + +At a high level, querying MAST catalogs follows three steps: + 1. Discover available collections and catalogs. + 2. Inspect catalog metadata to understand available fields and data types. + 3. Query the catalog using positional and/or criteria-based filters. -- The Hubble Source Catalog (HSC) -- The GALEX Catalog (V2 and V3) -- The Gaia (DR1 and DR2) and TGAS Catalogs -- The TESS Input Catalog (TIC) -- The TESS Candidate Target List (CTL) -- The Disk Detective Catalog -- The PanSTARRS Catalog (DR1 and DR2) -- The All-Sky PLATO Input Catalog (DR1) +Collections and Catalogs +======================== -Catalog Positional Queries -========================== +MAST catalogs are organized into **collections**, where each collection represents a related set of catalogs +with a shared scientific or mission context (for example, Hubble source catalogs, Gaia data releases, etc). +Within a collection, one or more **catalogs** may be available, each with its own schema and data. -Positional queries can be based on a sky position or a target name. -The returned fields vary by catalog, find the field documentation for specific catalogs -`here `__. -If no catalog is specified, the Hubble Source Catalog will be queried. +`~astroquery.mast.CatalogsClass` maintains a current collection and catalog as attributes. If no collection or catalog +is specified in a query, these attributes will be used as defaults. The ``collection`` attribute is an object +representing the current collection, and the ``catalog`` attribute is a string representing the name of the current +catalog within that collection. - .. doctest-remote-data:: >>> from astroquery.mast import Catalogs ... - >>> catalog_data = Catalogs.query_region("158.47924 -7.30962", catalog="Galex") - >>> print(catalog_data[:10]) # doctest: +IGNORE_OUTPUT - distance_arcmin objID survey ... fuv_flux_aper_7 fuv_artifact - ------------------ ------------------- ------ ... --------------- ------------ - 0.3493802506329695 6382034098673685038 AIS ... 0.047751952 0 - 0.7615422488595471 6382034098672634783 AIS ... -- 0 - 0.9243329366166956 6382034098672634656 AIS ... -- 0 - 1.162615739258038 6382034098672634662 AIS ... -- 0 - 1.2670891287503308 6382034098672634735 AIS ... -- 0 - 1.492173395497916 6382034098674731780 AIS ... 0.0611195639 0 - 1.6051235757244107 6382034098672634645 AIS ... -- 0 - 1.705418541388336 6382034098672634716 AIS ... -- 0 - 1.7463721100195875 6382034098672634619 AIS ... -- 0 - 1.7524423152919317 6382034098672634846 AIS ... -- 0 - - -Some catalogs have a maximum number of results they will return. -If a query results in this maximum number of results a warning will be displayed to alert -the user that they might be getting a subset of the true result set. + >>> print("Default collection:", Catalogs.collection.name) + Default collection: hsc + >>> print("Default catalog:", Catalogs.catalog) + Default catalog: dbo.SumMagAper2CatView + +These attributes may be changed at any time to set new defaults. Both ``collection`` and ``catalog`` will be validated +when set. When changing the collection, the catalog will be reset to the default for the new collection. .. doctest-remote-data:: - >>> from astroquery.mast import Catalogs - ... - >>> catalog_data = Catalogs.query_region("322.49324 12.16683", - ... catalog="HSC", - ... magtype=2) # doctest: +SHOW_WARNINGS - InputWarning: Coordinate string is being interpreted as an ICRS coordinate provided in degrees. - MaxResultsWarning: Maximum catalog results returned, may not include all sources within radius. - >>> print(catalog_data[:10]) - MatchID Distance MatchRA ... W3_F160W_MAD W3_F160W_N - --------- -------------------- ------------------ ... ------------ ---------- - 50180585 0.003984902849540913 322.4931746094701 ... nan 0 - 8150896 0.006357935819940561 322.49334740450234 ... nan 0 - 100906349 0.00808206428937523 322.4932839715549 ... nan 0 - 105434103 0.011947078376104195 322.49324000530777 ... nan 0 - 103116183 0.01274757103013683 322.4934207202404 ... nan 0 - 45593349 0.013026569623011767 322.4933878707698 ... nan 0 - 103700905 0.01306760650244682 322.4932769229944 ... nan 0 - 102470085 0.014611879195009472 322.49311034430366 ... nan 0 - 93722307 0.01476438046135455 322.49348351134466 ... nan 0 - 24781941 0.015234351867433582 322.49300148743345 ... nan 0 - -Radius is an optional parameter and the default is 0.2 degrees. + >>> Catalogs.collection = "TIC" # set collection to TESS Input Catalog + >>> print("New collection:", Catalogs.collection.name) + New collection: tic + >>> print("New catalog:", Catalogs.catalog) + New catalog: dbo.CatalogRecord -.. doctest-remote-data:: +Discovering Available Collections +--------------------------------- - >>> from astroquery.mast import Catalogs - ... - >>> catalog_data = Catalogs.query_object("M10", radius=.02, catalog="TIC") - >>> print(catalog_data[:10]) # doctest: +IGNORE_OUTPUT - ID ra dec ... wdflag dstArcSec - ---------- ---------------- ----------------- ... ------ ------------------ - 510188144 254.287449269816 -4.09954224264168 ... -1 0.7650443624931581 - 510188143 254.28717785824 -4.09908635292493 ... -1 1.3400566638148848 - 189844423 254.287799703996 -4.0994998249247 ... 0 1.3644407138867785 - 1305764031 254.287147439535 -4.09866105132406 ... -1 2.656905409847388 - 1305763882 254.286696117371 -4.09925522448626 ... -1 2.7561196688252894 - 510188145 254.287431890823 -4.10017293344746 ... -1 3.036238557555728 - 1305763844 254.286675148545 -4.09971617257086 ... 0 3.1424781549696217 - 1305764030 254.287249718516 -4.09841883152995 ... -1 3.365991083435227 - 1305764097 254.287599269103 -4.09837925361712 ... -1 3.4590276863989 - 1305764215 254.28820865799 -4.09859677020253 ... -1 3.7675526728257034 - - -The Hubble Source Catalog, the Gaia Catalog, and the PanSTARRS Catalog have multiple versions. -An optional version parameter allows you to select which version you want, the default is the highest version. +To list all available catalog collections hosted at MAST, use the +`~astroquery.mast.CatalogsClass.get_collections` method. -.. doctest-remote-data:: +This returns an `~astropy.table.Table` containing the names of all available collections. - >>> catalog_data = Catalogs.query_region("158.47924 -7.30962", radius=0.1, - ... catalog="Gaia", version=2) - >>> print("Number of results:",len(catalog_data)) - Number of results: 111 - >>> print(catalog_data[:4]) - solution_id designation ... distance - ------------------- ---------------------------- ... ------------------ - 1635721458409799680 Gaia DR2 3774902350511581696 ... 0.6326770410972467 - 1635721458409799680 Gaia DR2 3774901427093274112 ... 0.8440033390947586 - 1635721458409799680 Gaia DR2 3774902148648277248 ... 0.9199206487344911 - 1635721458409799680 Gaia DR2 3774902453590798208 ... 1.3578181104319944 - -The PanSTARRS Catalog has multiple data releases as well as multiple queryable tables. -An optional data release parameter allows you to select which data release is desired, with the default being the latest version (dr2). -The table to query is a required parameter. +Some historical collections are no longer supported for querying, and will not appear in this list. If a collection +has been renamed or deprecated, Astroquery will issue a warning and suggest the appropriate replacement where possible. .. doctest-remote-data:: - >>> catalog_data = Catalogs.query_region("158.47924 -7.30962", - ... radius=0.01, - ... catalog="Panstarrs", - ... data_release="dr1", - ... table="mean") - >>> print("Number of results:",len(catalog_data)) # doctest: +IGNORE_OUTPUT - Number of results: 42 - >>> print(catalog_data[:5]) # doctest: +IGNORE_OUTPUT - objName objAltName1 ... yFlags distance - -------------------------- ----------- ... ------ --------------------- - PSO J103357.027-071828.380 -999 ... 0 0.008463641060218161 - PSO J103357.130-071834.314 -999 ... 4120 0.008734008139467626 - PSO J103357.065-071832.617 -999 ... 4120 0.008480972171475138 - PSO J103355.542-071833.037 -999 ... 16416 0.0022151507196657037 - PSO J103356.363-071839.939 -999 ... 0 0.005754569818470991 - -Catalog Criteria Queries -======================== + >>> collections = Catalogs.get_collections() + >>> print(collections) # doctest: +IGNORE_OUTPUT + collection_name + --------------- + caom + classy + gaiadr3 + hsc + hscv2 + missionmast + ps1dr1 + ps1dr2 + ps1_dr2 + registry + skymapperdr4 + tic + ullyses + goods + candels + 3dhst + deepspace + +Discovering Catalogs in a Collection +------------------------------------- + +Once a collection is selected, you can list the catalogs available within it using the +`~astroquery.mast.CatalogsClass.get_catalogs` method. + +To query catalogs for a specific collection without changing the class state, you can pass the +collection name as an argument to the method. + +.. doctest-remote-data:: -The TESS Input Catalog (TIC), Disk Detective Catalog, and PanSTARRS Catalog can also be queried based on non-positional criteria. + >>> catalogs = Catalogs.get_catalogs('hsc') + >>> catalogs.pprint(max_width=-1) # doctest: +IGNORE_OUTPUT + catalog_name description + -------------------------- ------------------------------------------------------- + tap_schema.schemas description of schemas in this dataset + tap_schema.tables description of tables in this dataset + tap_schema.columns description of columns in this dataset + tap_schema.keys description of foreign keys in this dataset + tap_schema.key_columns description of foreign key columns in this dataset + dbo.detailedcatalog Detailed list of source catalog parameters + dbo.hcvdetailedview Detailed list of Hubble Catalog of Variables parameters + dbo.hcvsummaryview Summary list of Hubble Catalog of Variables parameters + dbo.propermotionsview List of proper motion information + dbo.sourcepositionsview List of source position information + dbo.summagaper2catview Summary list of source catalog with Aper2 magnitudes + dbo.summagautocatview Summary list of source catalog with MagAuto magnitudes + dbo.catalog_image_metadata Summary list of Image processing metadata + +Inspecting Catalog Metadata +---------------------------- + +Before querying a catalog, it is often useful to inspect its metadata to understand the available columns, +data types, and supported query capabilities. The `~astroquery.mast.CatalogsClass.get_column_metadata` method returns +an `~astropy.table.Table` describing the catalog schema, including column names, data types, units, and descriptions. +This metadata can help you construct valid queries, select columns of interest, and understand which fields support +numeric comparisons, string matching, or spatial queries. + +Again, you can specify a collection and catalog explicitly as inputs to the function, or use the current defaults. +If you only specify a collection, the default catalog for that collection will be used. .. doctest-remote-data:: >>> from astroquery.mast import Catalogs ... - >>> catalog_data = Catalogs.query_criteria(catalog="Tic", Bmag=[30,50], objType="STAR") - >>> print(catalog_data) # doctest: +IGNORE_OUTPUT - ID version HIP TYC ... e_Dec_orig raddflag wdflag objID - --------- -------- --- --- ... ------------------ -------- ------ ---------- - 125413929 20190415 -- -- ... 0.293682765259495 1 0 579825059 - 261459129 20190415 -- -- ... 0.200397148604244 1 0 1701625107 - 64575709 20190415 -- -- ... 0.21969663115091 1 0 595775997 - 94322581 20190415 -- -- ... 0.205286802302475 1 0 606092549 - 125414201 20190415 -- -- ... 0.22398993783274 1 0 579825329 - 463721073 20190415 -- -- ... 0.489828592248652 -1 1 710312391 - 81609218 20190415 -- -- ... 0.146788572369267 1 0 630541794 - 282024596 20190415 -- -- ... 0.548806522539047 1 0 573765450 - 23868624 20190415 -- -- ... 355.949 -- 0 916384285 - 282391528 20190415 -- -- ... 0.47766300834538 0 0 574723760 - 123585000 20190415 -- -- ... 0.618316068787371 0 0 574511442 - 260216294 20190415 -- -- ... 0.187170498094167 1 0 683390717 - 406300991 20190415 -- -- ... 0.0518318978617112 0 0 1411465651 + >>> catalog_metadata = Catalogs.get_column_metadata('gaiadr3', 'dbo.gaia_source') + >>> catalog_metadata[:5].pprint(max_width=-1) + column_name datatype unit ucd description + ------------ -------- ---- ----------------- ---------------------------- + solution_id long meta.version catalog version + designation char meta.id;meta.main designation + source_id long meta.id source id + random_index long meta.code random index + ref_epoch double yr time.epoch reference epoch julian years -.. doctest-remote-data:: +Querying Catalogs +================== - >>> from astroquery.mast import Catalogs - ... - >>> catalog_data = Catalogs.query_criteria(catalog="Ctl", - ... object_name='M101', - ... radius=1, - ... Tmag=[10.75,11]) - >>> print(catalog_data) - ID version HIP TYC ... raddflag wdflag objID - --------- -------- --- ------------ ... -------- ------ --------- - 233458861 20190415 -- 3852-01407-1 ... 1 0 150390757 - 441662028 20190415 -- 3855-00941-1 ... 1 0 150395533 - 441658008 20190415 -- 3852-00116-1 ... 1 0 150246361 - 441639577 20190415 -- 3852-00429-1 ... 1 0 150070672 - 441658179 20190415 -- 3855-00816-1 ... 1 0 150246482 - 154258521 20190415 -- 3852-01403-1 ... 1 0 150281963 - 441659970 20190415 -- 3852-00505-1 ... 1 0 150296707 - 441660006 20190415 -- 3852-00341-1 ... 1 0 150296738 +The Catalogs interface provides three closely related query methods. All three methods return results from a MAST +catalog as an `~astropy.table.Table` (or a scalar count, if requested), and all three support column-based filtering, +sorting, and result limiting. The primary difference between them is how positional constraints are specified. + +At a high level: + - `~astroquery.mast.CatalogsClass.query_criteria` is the most flexible method. It supports purely column-based queries, + purely positional queries, or a combination of both. + + - `~astroquery.mast.CatalogsClass.query_region` is a convenience wrapper for positional queries using coordinates or an explicit region. + + - `~astroquery.mast.CatalogsClass.query_object` is a convenience wrapper for cone searches centered on a resolved object name. + +All three methods ultimately construct and execute an ADQL query against the MAST TAP service. +Shared Query Parameters +------------------------ + +The following parameters are supported by all three query methods: + - ``collection`` : The catalog collection to query. If not specified, the value of the instance's ``collection`` attribute is used. + + - ``catalog`` : The catalog within the collection to query. If not specified, the value of the instance's ``catalog`` attribute is used, or if ``collection`` is specified, the default catalog for that collection. + + - ``limit`` : Maximum number of rows to return (default: 5000). + + - ``offset`` : Number of rows to skip before returning results (default: 0). + + - ``count_only`` : If True, return only the number of matching records instead of the records themselves. + + - ``select_cols`` : A list of column names to include in the result. If omitted, all columns are returned. + + - ``sort_by``: One or more column names to sort by. + + - ``sort_desc`` : Whether to sort in descending order (either a single boolean applied to all ``sort_by`` columns or one per column). + +These parameters allow users to control the scope and format of their queries consistently across all three methods. + +Writing Queries +---------------- + +The `~astroquery.mast.CatalogsClass.query_criteria` method supports both positional parameters and column-based filters. +Positional constraints are optional. + +Supported positional parameters include: + - ``coordinates`` : Sky coordinates around which to perform a cone search. + - ``object_name`` : Name of the object around which to perform a cone search. + - ``resolver`` : Resolver service to use for object name resolution. + - ``radius`` : Radius of the cone search around the specified coordinates or object name. Can be defined as an `~astropy.units.Quantity`, a string with units (e.g., ``"10 arcsec"``), or a numeric value interpreted as degrees. + - ``region`` : Explicit region specification for the search. See :ref:`specifying-spatial-regions` below for more details. + +If no positional parameters are supplied, the query is purely criteria-based. If positional parameters are supplied, +they are combined with any column-based criteria using logical **AND**. .. doctest-remote-data:: >>> from astroquery.mast import Catalogs ... - >>> catalog_data = Catalogs.query_criteria(catalog="DiskDetective", - ... object_name="M10", - ... radius=2, - ... state="complete") - >>> print(catalog_data) # doctest: +IGNORE_OUTPUT - designation ... ZooniverseURL - ------------------- ... ---------------------------------------------------- - J165628.40-054630.8 ... https://talk.diskdetective.org/#/subjects/AWI0005cka - J165748.96-054915.4 ... https://talk.diskdetective.org/#/subjects/AWI0005ckd - J165427.11-022700.4 ... https://talk.diskdetective.org/#/subjects/AWI0005ck5 - J165749.79-040315.1 ... https://talk.diskdetective.org/#/subjects/AWI0005cke - J165327.01-042546.2 ... https://talk.diskdetective.org/#/subjects/AWI0005ck3 - J165949.90-054300.7 ... https://talk.diskdetective.org/#/subjects/AWI0005ckk - J170314.11-035210.4 ... https://talk.diskdetective.org/#/subjects/AWI0005ckv - - -The `~astroquery.mast.CatalogsClass.query_criteria` function requires at least one non-positional parameter. -These parameters are the column names listed in the `field descriptions `__ -of the catalog being queried. They do not include object_name, coordinates, or radius. Running a query with only positional -parameters will result in an error. + >>> result = Catalogs.query_criteria(collection='hsc', + ... coordinates="322.49324 12.16683", + ... radius='2 arcsec', + ... sort_by=['numimages', 'starttime'], + ... sort_desc=[False, True], + ... limit=5, + ... select_cols=['matchid', 'matchra', 'matchdec', 'numimages', 'starttime']) + >>> result.pprint(max_width=-1) + matchid matchra matchdec numimages starttime + deg deg + --------- ------------------ ------------------ --------- -------------------------- + 100906349 322.4932839715549 12.166957658789572 1 2013-09-01 14:42:57.487000 + 37053748 322.49333237602906 12.167170257824768 1 2013-09-01 14:09:53.487000 + 61895629 322.493715383149 12.166629788750484 1 2011-10-22 08:10:21.217000 + 19779150 322.49277386636714 12.166728768957904 2 2013-09-01 14:09:53.487000 + 11562863 322.49294957070185 12.166668540816076 2 2006-05-02 01:13:43.920000 + +To filter results based on column values, users may specify criteria as keyword arguments, +where the keyword is the column name and the value is the desired filter. Multiple criteria are combined +using logical **AND**. + +Criteria syntax supports a variety of operations, including: + +- Exact matches by specifying a value for a column. + +- To filter by multiple values for a single column, use a list of values. This performs an **OR** operation between the values. + +- A filter value can be negated by prefixing it with ``!``, meaning that rows matching that value will be excluded from the results. + When a negated value is present in a list of filters, any positive values in that set are combined with **OR** logic, and any negated + values are combined with **AND** logic against the positives. + +- For columns with a numeric data type, filter using comparison values (``<``, ``>``, ``<=``, ``>=``). + + - ``<``: Return values less than or before the given number/date + + - ``>``: Return values greater than or after the given number/date + + - ``<=``: Return values less than or equal to the given number/date + + - ``>=``: Return values greater than or equal to the given number/date + +- For columns with a numeric data type, select an inclusive range with the syntax ``'#..#'``. + +- Wildcards are special characters used in search patterns to represent one or more unknown characters, + allowing for flexible matching of strings. The wildcard characters are ``*`` and ``%`` and they replace any number + of characters preceding, following, or in between existing characters, depending on their placement. .. doctest-remote-data:: - >>> from astroquery.mast import Catalogs - ... - >>> catalog_data = Catalogs.query_criteria(catalog="Tic", - ... object_name='M101', radius=1) - Traceback (most recent call last): - ... - astroquery.exceptions.InvalidQueryError: At least one non-positional criterion must be supplied. + >>> result = Catalogs.query_criteria(collection='ullyses', + ... catalog='sciencemetadata', + ... target_name_ullyses='NGC*', + ... target_classification='!Galaxy', + ... known_binary=False, + ... sp_class=['O', 'B'], + ... gaia_parallax=['<-0.1', '>=0'], + ... star_teff='30000..50000', + ... select_cols=['target_name_ullyses', 'target_classification', 'known_binary', 'sp_class', 'gaia_parallax', 'star_teff']) + >>> result.pprint(max_width=-1) + target_name_ullyses target_classification known_binary sp_class gaia_parallax star_teff + mas K + ------------------- --------------------- ------------ -------- ------------- --------- + NGC346 ELS 043 Early B Dwarf False B -0.111579 33000.0 + NGC346 MPG 487 Late O Dwarf False O -0.389724 35800.0 + NGC 3109 EBU 20 Mid O Supergiant False O 0.876327 31150.0 + NGC 3109 EBU 34 Mid O Supergiant False O -0.126069 33050.0 -The PanSTARRS catalog also accepts additional parameters to allow for query refinement. These options include column selection, -sorting, column criteria, page size and page number. Additional information on PanSTARRS queries may be found -`here `__. +The `~astroquery.mast.CatalogsClass.query_region` and `~astroquery.mast.CatalogsClass.query_object` methods are +convenience wrappers around `~astroquery.mast.CatalogsClass.query_criteria`: -Columns returned from the query may be submitted with the columns parameter as a list of column names. + - `~astroquery.mast.CatalogsClass.query_region` requires a positional constraint (``coordinates`` or ``region``). -The query may be sorted with the sort_by parameter composed of either a single column name (to sort ascending), -or a list of multiple column names and/or tuples of direction and column name (ASC/DESC, column name). + - `~astroquery.mast.CatalogsClass.query_object` requires an ``object_name`` and performs a cone search. -To filter the query, criteria per column name are accepted. The 'AND' operation is performed between all -column name criteria, and the 'OR' operation is performed within column name criteria. Per each column name -parameter, criteria may consist of either a value or a list. The list may consist of a mix of values and -tuples of criteria decorator (min, gte, gt, max, lte, lt, like, contains) and value. +Both methods also accept column-based criteria, which are applied in the same way as in `~astroquery.mast.CatalogsClass.query_criteria`. .. doctest-remote-data:: - >>> catalog_data = Catalogs.query_criteria(coordinates="5.97754 32.53617", - ... radius=0.01, - ... catalog="PANSTARRS", - ... table="mean", - ... data_release="dr2", - ... nStackDetections=[("gte", 2)], - ... columns=["objName", "objID", "nStackDetections", "distance"], - ... sort_by=[("desc", "distance")], - ... pagesize=15) - >>> print(catalog_data[:10]) # doctest: +IGNORE_OUTPUT - objName objID nStackDetections distance - --------------------- ------------------ ---------------- --------------------- - PSO J005.9812+32.5270 147030059812483022 5 0.009651200148871086 - PSO J005.9726+32.5278 147030059727583992 2 0.0093857181370567 - PSO J005.9787+32.5453 147050059787164914 4 0.009179045509852305 - PSO J005.9722+32.5418 147050059721440704 4 0.007171813230776031 - PSO J005.9857+32.5377 147040059855825725 4 0.007058815429178634 - PSO J005.9810+32.5424 147050059809651427 2 0.006835678269917365 - PSO J005.9697+32.5368 147040059697224794 2 0.006654002479439699 - PSO J005.9712+32.5330 147040059711340087 4 0.006212461367287632 - PSO J005.9747+32.5413 147050059747400181 5 0.0056515210592035965 - PSO J005.9775+32.5314 147030059774678271 3 0.004739286624336443 - - -Hubble Source Catalog (HSC) specific queries -============================================ - -Given an HSC Match ID, return all catalog results. + >>> result = Catalogs.query_region(collection='skymapperdr4', + ... coordinates="158.47924 -7.30962", + ... radius='1 arcmin', + ... select_cols=['object_id', 'raj2000', 'dej2000']) + >>> result.pprint(max_width=-1) + object_id raj2000 dej2000 + deg deg + ---------- ---------- --------- + 1116262239 158.475216 -7.29985 + 92307496 158.483015 -7.323183 + 92307428 158.46783 -7.319955 + 2151499732 158.466411 -7.319867 .. doctest-remote-data:: - >>> from astroquery.mast import Catalogs - ... - >>> catalog_data = Catalogs.query_object("M10", - ... radius=.001, - ... catalog="HSC", - ... magtype=1) - >>> matchid = catalog_data[0]["MatchID"] - >>> print(matchid) - 7542452 - >>> matches = Catalogs.query_hsc_matchid(matchid) - >>> print(matches) - CatID MatchID ... cd_matrix - --------- ------- ... ------------------------------------------------------ - 419094794 7542452 ... -1.10056e-005 5.65193e-010 5.65193e-010 1.10056e-005 - 419094795 7542452 ... -1.10056e-005 5.65193e-010 5.65193e-010 1.10056e-005 - 401289578 7542452 ... -1.10056e-005 1.56577e-009 1.56577e-009 1.10056e-005 - 401289577 7542452 ... -1.10056e-005 1.56577e-009 1.56577e-009 1.10056e-005 - 257194049 7542452 ... -1.38889e-005 -5.26157e-010 -5.26157e-010 1.38889e-005 - 257438887 7542452 ... -1.38889e-005 -5.26157e-010 -5.26157e-010 1.38889e-005 - - -HSC spectra accessed through this class as well. `~astroquery.mast.CatalogsClass.get_hsc_spectra` -does not take any arguments, and simply loads all HSC spectra. + >>> result = Catalogs.query_object(collection='skymapperdr4', + ... object_name='M11', + ... radius=0.01, + ... catwise_id='2825m061*', + ... sort_by='mean_epoch', + ... select_cols=['object_id', 'raj2000', 'dej2000', 'catwise_id', 'mean_epoch']) + >>> result[:5].pprint(max_width=-1) + object_id raj2000 dej2000 catwise_id mean_epoch + deg deg d + ---------- ---------- --------- ------------------ ---------- + 277772662 282.75865 -6.273223 2825m061_b0-113841 56833.6242 + 277772668 282.762729 -6.274544 2825m061_b0-010099 56834.1188 + 2389914703 282.768791 -6.275556 2825m061_b0-045485 56834.6156 + 277772658 282.758508 -6.276218 2825m061_b0-049457 56834.6167 + 277772511 282.764593 -6.27897 2825m061_b0-046693 56895.2497 + +.. _specifying-spatial-regions: + +Specifying Spatial Regions +-------------------------- + +Catalogs that support spatial queries allow regions to be specified in several ways. + +Cone Searches +^^^^^^^^^^^^^ + +Cone searches are the most common type of spatial query, defined by a center position and a radius. They may be specified using: + - ``coordinates`` and ``radius`` + - ``object_name`` and ``radius`` + - A `~regions.CircleSkyRegion` object as the ``region`` parameter + - A Space-Time Coordinate (STC) CIRCLE region string as the ``region`` parameter + +An STC-S CIRCLE region string has the following format: + +``CIRCLE [frame] `` + +For example: + +``CIRCLE ICRS 210.8 54.35 0.05`` + +This means: + - ICRS: Coordinate frame (optional; defaults to ICRS if omitted) + - 210.8: Right Ascension in degrees + - 54.35: Declination in degrees + - 0.05: Radius in degrees .. doctest-remote-data:: + >>> result = Catalogs.query_region(collection='ps1_dr2', + ... region='CIRCLE ICRS 18.895 -6.944 0.01', + ... sort_by='objID', + ... select_cols=['objName', 'objID', 'raMean', 'decMean']) + >>> result[:5].pprint(max_width=-1) + objName objID raMean decMean + deg deg + ----------------------- ----------------- ----------------- ------------------ + PSX J011534.22-065706.1 99650188926188340 18.89261844 -6.95171552 + PSX J011534.90-065704.8 99650188954378784 18.89543744 -6.95134445 + PSX J011533.84-065651.9 99660188909803092 18.89100927680362 -6.947762708324004 + PSX J011533.97-065643.6 99660188915395866 18.8915527 -6.9454449 + PSX J011534.30-065636.0 99660188931668392 18.89294147 -6.94333417 - >>> from astroquery.mast import Catalogs - ... - >>> all_spectra = Catalogs.get_hsc_spectra() - >>> print(all_spectra[:10]) - ObjID DatasetName MatchID ... PropID HSCMatch - ----- -------------------------------------------- -------- ... ------ -------- - 20010 HAG_J072655.67+691648.9_J8HPAXAEQ_V01.SPEC1D 19657846 ... 9482 Y - 20011 HAG_J072655.69+691648.9_J8HPAOZMQ_V01.SPEC1D 19657846 ... 9482 Y - 20012 HAG_J072655.76+691729.7_J8HPAOZMQ_V01.SPEC1D 19659745 ... 9482 Y - 20013 HAG_J072655.82+691620.0_J8HPAOZMQ_V01.SPEC1D 19659417 ... 9482 Y - 20014 HAG_J072656.34+691704.7_J8HPAXAEQ_V01.SPEC1D 19660230 ... 9482 Y - 20015 HAG_J072656.36+691704.7_J8HPAOZMQ_V01.SPEC1D 19660230 ... 9482 Y - 20016 HAG_J072656.36+691744.9_J8HPAOZMQ_V01.SPEC1D 19658847 ... 9482 Y - 20017 HAG_J072656.37+691630.2_J8HPAXAEQ_V01.SPEC1D 19660827 ... 9482 Y - 20018 HAG_J072656.39+691630.2_J8HPAOZMQ_V01.SPEC1D 19660827 ... 9482 Y - 20019 HAG_J072656.41+691734.9_J8HPAOZMQ_V01.SPEC1D 19656620 ... 9482 Y - - -Individual or ranges of spectra can be downloaded using the -`~astroquery.mast.CatalogsClass.download_hsc_spectra` function. +Polygon Searches +^^^^^^^^^^^^^^^^^ + +Polygon searches allow for more complex spatial queries by defining a polygonal region on the sky. +They may be specified using any of the following as the ``region`` parameter: + + - An iterable of coordinate pairs + - A `~regions.PolygonSkyRegion` object + - A Space-Time Coordinate (STC) POLYGON region string + +An STC-S POLYGON string has the form: + +``POLYGON [frame] ...`` + +For example: + +``POLYGON ICRS 210.7 54.3 210.9 54.3 210.9 54.4 210.7 54.4`` + +This defines a four-vertex polygon with vertices given as (RA, Dec) pairs in degrees. At least three vertices (six numbers) are required. .. doctest-remote-data:: + >>> result = Catalogs.query_criteria(collection='caom', + ... region='POLYGON ICRS 18.85 -6.95 18.86 -6.95 18.86 -6.94 18.85 -6.94', + ... limit=5, + ... select_cols=['target_name', 'obs_id', 's_ra', 's_dec']) + >>> result.pprint(max_width=-1) + target_name obs_id s_ra s_dec + deg deg + ----------- ------------------------------------------------------------------------- ----------------- ------------------ + 408084461 hlsp_tglc_tess_ffi_gaiaid-2475555794352572672-s0003-cam1-ccd3_tess_v1_llc 18.85365901920825 -6.941643227687541 + 408084461 hlsp_t16_tess_ffi_s0003-cam1-ccd3-02475555794352572672_tess_v01 18.8536465402 -6.9416358523 + 408084461 hlsp_tasoc_tess_ffi_tic00408084461-s0003-cam1-ccd3-c1800_tess_v05 18.8532579321195 -6.94140657784404 + 408084461 hlsp_gsfc-eleanor-lite_tess_ffi_s0003-0000000408084461_tess_v1.0_lc 18.8532579321195 -6.941406577844042 + 408084461 hlsp_qlp_tess_ffi_s0097-0000000408084461_tess_v01_llc 18.8532579321 -6.94140657784 + + +Counting Results +----------------- + +All query methods support a ``count_only=True`` option, which returns only the number of matching records: - >>> from astroquery.mast import Catalogs - ... - >>> all_spectra = Catalogs.get_hsc_spectra() - >>> manifest = Catalogs.download_hsc_spectra(all_spectra[100:104]) # doctest: +IGNORE_OUTPUT - Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.61+691530.3_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.61+691530.3_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] - Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.68+691535.9_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.68+691535.9_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] - Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.70+691530.2_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.70+691530.2_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] - Downloading URL https://hla.stsci.edu/cgi-bin/ecfproxy?file_id=HAG_J072704.73+691808.0_J8HPAOZMQ_V01.SPEC1D.fits to ./mastDownload/HSC/HAG_J072704.73+691808.0_J8HPAOZMQ_V01.SPEC1D.fits ... [Done] - ... - >>> print(manifest) # doctest: +IGNORE_OUTPUT - Local Path ... URL - -------------------------------------------------------------------- ... ---- - ./mastDownload/HSC/HAG_J072704.61+691530.3_J8HPAOZMQ_V01.SPEC1D.fits ... None - ./mastDownload/HSC/HAG_J072704.68+691535.9_J8HPAOZMQ_V01.SPEC1D.fits ... None - ./mastDownload/HSC/HAG_J072704.70+691530.2_J8HPAOZMQ_V01.SPEC1D.fits ... None - ./mastDownload/HSC/HAG_J072704.73+691808.0_J8HPAOZMQ_V01.SPEC1D.fits ... None \ No newline at end of file +.. doctest-remote-data:: + + >>> count = Catalogs.query_criteria(collection='skymapperdr4', + ... region=[(20, -5), (20, -6), (21, -6), (21, -5)], + ... count_only=True) + >>> print('Number of matching records:', count) + Number of matching records: 5479 + +This is useful for estimating result sizes before executing large queries. + +Deprecated Interfaces +====================== + +Several legacy methods related to the Hubble Source Catalog (HSC) remain available but are deprecated and will be removed +in a future release. These methods include: + +- `~astroquery.mast.CatalogsClass.query_hsc_matchid` +- `~astroquery.mast.CatalogsClass.get_hsc_spectra` +- `~astroquery.mast.CatalogsClass.download_hsc_spectra` + +New workflows should use the general `~astroquery.mast.CatalogsClass` interface described above.