diff --git a/docs/ObservationProperties.md b/docs/ObservationProperties.md index dace8ffd4..1ff7d4852 100644 --- a/docs/ObservationProperties.md +++ b/docs/ObservationProperties.md @@ -46,7 +46,7 @@ Cal Bench filter 1 position. Throughput = 10^-OD. Values: OD 0.1, OD 1.0, OD 1.3, OD 2.0, OD 3.0, OD 4.0 **CalND2**: `str` - Cal Bench filter 2 position. Throughput = 10^-OD. Values: OD 0.1, OD 0.3, OD 0.5, OD 0.8, OD 1.0, OD 4.0 + Cal Bench filter 2 position. Throughput = 10^-OD. Values: OD 0.1, OD 0.3, OD 0.5, OD 1.0, OD 1.3, OD 2.0 **NodN**: `float` [arcseconds] Distance to nod the telescope North before starting exposure. diff --git a/docs/ZeroPointAnalysis/1-PrepareDataFile/1-PrepareDataFile.md b/docs/ZeroPointAnalysis/1-PrepareDataFile/1-PrepareDataFile.md new file mode 100644 index 000000000..71f571d66 --- /dev/null +++ b/docs/ZeroPointAnalysis/1-PrepareDataFile/1-PrepareDataFile.md @@ -0,0 +1,166 @@ +# Examine Properties of Data Set + +This data set includes (anonymized) results of the first few years of KPF observations. This contains select FITS header keyworsds from the DRP's L1 output file. We're primarily interested in the DRP's estimate of the signal to noise at various wavelengths (the `SNRSCnnn` values). These are the typical signal to noise for one order around the specified wavelength value in nm. + +In this notebook, we anonymize the data then filter out data with certain known issues. After filtering we have over 12,000 individual observations (FITS files). Finally, we examine the properties of the targets (effective temperature and Gaia G magnituide), of the observations (time of observation), and of the DRP reductions (DRP version used). + + +```python +from pathlib import Path +import numpy as np +from astropy.table import Table, MaskedColumn +from astropy.time import Time + +from matplotlib import pyplot as plt +``` + + +```python +# Read raw data +raw_data_file = Path('data/Feb2026.csv') +``` + + +```python +if raw_data_file.exists(): + t = Table.read(raw_data_file, format='ascii.csv') + t.remove_columns(['ObsID', 'OBJECT', 'TARGNAME', 'TOTCNTS', 'CCFRV_MPS', 'CCD1RV_MPS', 'CCD2RV_MPS', 'CCFERV_MPS', 'CCFBJD', 'CCFERV']) + # Separate out TOTCORR values to separate columns + TOTCORR_1 = MaskedColumn(data=np.zeros(len(t)), name='TOTCORR_1') + TOTCORR_2 = MaskedColumn(data=np.zeros(len(t)), name='TOTCORR_2') + TOTCORR_3 = MaskedColumn(data=np.zeros(len(t)), name='TOTCORR_3') + TOTCORR_4 = MaskedColumn(data=np.zeros(len(t)), name='TOTCORR_4') + TOTCORR_SUM = MaskedColumn(data=np.zeros(len(t)), name='TOTCORR_SUM') + for i,row in enumerate(t): + masked = t['TOTCORR'].mask[i] + if not masked: + TOTCORR_1[i] = t['TOTCORR'][i].split()[0] + TOTCORR_2[i] = t['TOTCORR'][i].split()[1] + TOTCORR_3[i] = t['TOTCORR'][i].split()[2] + TOTCORR_4[i] = t['TOTCORR'][i].split()[3] + TOTCORR_SUM[i] = np.sum([float(v) for v in t['TOTCORR'][i].split()]) + TOTCORR_1.mask = t['TOTCORR'].mask + TOTCORR_2.mask = t['TOTCORR'].mask + TOTCORR_3.mask = t['TOTCORR'].mask + TOTCORR_4.mask = t['TOTCORR'].mask + TOTCORR_SUM.mask = t['TOTCORR'].mask + t.add_columns([TOTCORR_1, TOTCORR_2, TOTCORR_3, TOTCORR_4, TOTCORR_SUM]) +``` + + +```python +# Remove data with known bad parameters +if raw_data_file.exists(): + # Remove Data with MJD-OBS Before KPF was Installed + t = t[t['MJD-OBS'] > 59200] + print(len(t)) + # Remove Data with Gmag<=0 + t = t[t['GAIAMAG'] > 0] + print(len(t)) + #because of missing labeling in OBs, this was a default value, so remove all stars with this value as the data is not usable. + t = t[(t['GAIAMAG'] < 9.2) | (t['GAIAMAG'] > 9.5)] + print(len(t)) + # Remove very short exposures (removes start state errors) + t = t[t['ELAPSED'] > 7] + print(len(t)) +``` + + 13547 + 12700 + 12118 + 12058 + + + +```python +anonymized_data_file = Path('data/Feb2026_anonymized.csv') +if raw_data_file.exists(): + t.write(anonymized_data_file, overwrite=True) +else: + t = Table.read(anonymized_data_file, format='ascii.csv') +``` + +# Examine Properties of Data Set + + +```python +# KPFERA 1.0 is before SM1 +SM1start = Time('2024-02-03') +SM1end = Time('2024-02-23') +# KPFERA 2.0 is between SM1 and SM2 +SM2start = Time('2024-10-31') +SM2end = Time('2024-11-20') +# KPFERA 2.5/2.6 is between SM2 and SM3 +SM3start = Time('2025-03-28') +SM3end = Time('2025-04-23') +# KPFERA 3.0 is between SM3 and SM4 +SM4start = Time('2025-08-31') +SM4end = Time('2025-10-27') +# KPFERA 4.0 is after SM4 +``` + + +```python +plt.figure(figsize=(8,8)) + +plt.subplot(3,1,1) +plt.title('Histogram of Teff and Gmag Values in the Data Set') +bins=np.arange(2700,10000,100) +plt.hist(t['TARGTEFF'], bins=bins, log=True) +plt.axvline(2700, color='r', label='KPF-etc Limits') +plt.axvline(6600, color='r') +plt.xlabel('Teff') +plt.legend(loc='best') + +plt.subplot(3,1,2) +plt.hist(t['GAIAMAG'], bins=40, log=True) +plt.axvline(2, color='r', label='KPF-etc Limits') +plt.axvline(19, color='r') +plt.xlabel('Gmag') + +plt.subplot(3,1,3) +nd = max(t['MJD-OBS'])-min(t['MJD-OBS']) +plt.hist(t['MJD-OBS'], bins=int(nd/7)) +plt.axvspan(SM1start.mjd, SM1end.mjd, color='k', alpha=0.3, label='Servicing Mission') +plt.axvspan(SM2start.mjd, SM2end.mjd, color='k', alpha=0.3) +plt.axvspan(SM3start.mjd, SM3end.mjd, color='k', alpha=0.3) +plt.axvspan(SM4start.mjd, SM4end.mjd, color='k', alpha=0.3) +plt.xlabel('MJD') + +plt.show() +``` + + + +![png](output_8_0.png) + + + + +```python +drpvs = set([str(v) for v in t['DRPTAGL2']]) +print("DRPTAGL2 : N Files") +print("____________________") +total = 0 +for v in sorted(drpvs): + if v == '--': + Nv = np.sum(np.array(t['DRPTAGL2'].mask, dtype=int)) + else: + tv = t[t['DRPTAGL2'] == v] + Nv = len(tv) + total += Nv + print(f"{v:11s}: {Nv}") +print(f"TOTAL = {total} ({len(t)})") +``` + + DRPTAGL2 : N Files + ____________________ + -- : 38 + v2.10.3 : 3216 + v2.11.1 : 23 + v2.12.0rc1 : 4369 + v2.5.3 : 35 + v2.8.2 : 2030 + v2.9.1 : 2347 + TOTAL = 12058 (12058) + diff --git a/docs/ZeroPointAnalysis/1-PrepareDataFile/output_8_0.png b/docs/ZeroPointAnalysis/1-PrepareDataFile/output_8_0.png new file mode 100644 index 000000000..512c45d8e Binary files /dev/null and b/docs/ZeroPointAnalysis/1-PrepareDataFile/output_8_0.png differ diff --git a/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/2-ExpMeterZeroPoint.md b/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/2-ExpMeterZeroPoint.md new file mode 100644 index 000000000..7fae4b3ce --- /dev/null +++ b/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/2-ExpMeterZeroPoint.md @@ -0,0 +1,249 @@ +# Determine Exposure Meter Zero Points + +Each observation can be used as a measure of the zero point of the exposure meter. By examining the distribution of these zero point values, we can try to determine the nominal zero point -- the zero point we would achieve under nominal (i.e. good) conditions. We'll take the 90th percentile of the distribution as "nominal" for this calculation. + + +```python +from pathlib import Path +import yaml +import numpy as np +from astropy import stats +from astropy.time import Time +from astropy.table import Table, MaskedColumn +from kpf_etc.etc import kpf_photon_noise_estimate + +import matplotlib.pyplot as plt + +data_file = 'data/Feb2026_anonymized.csv' +t = Table.read(data_file, format='ascii.csv') +``` + + +```python +# KPFERA 1.0 is before SM1 +SM1start = Time('2024-02-03') +SM1end = Time('2024-02-23') +# KPFERA 2.0 is between SM1 and SM2 +SM2start = Time('2024-10-31') +SM2end = Time('2024-11-20') +# KPFERA 2.5/2.6 is between SM2 and SM3 +SM3start = Time('2025-03-28') +SM3end = Time('2025-04-23') +# KPFERA 3.0 is between SM3 and SM4 +SM4start = Time('2025-08-31') +SM4end = Time('2025-10-27') +# KPFERA 4.0 is after SM4 +``` + + +```python +def Nmasked(t, column='TOTCORR'): + mask_count = int(np.sum(np.array(t[column].mask, dtype=int))) + print(f"{mask_count} masked out of {len(t)} for {column}") +``` + + +```python +Nmasked(t) +``` + + 2004 masked out of 12058 for TOTCORR + + + +```python +# Mask out negative exposure meter flux +t['TOTCORR'].mask = t['TOTCORR'].mask | (t['TOTCORR_SUM'] <= 0).data +Nmasked(t) +``` + + 3651 masked out of 12058 for TOTCORR + + + +```python +# Mask out exposure meter data from before reformatter realignment +t['TOTCORR'].mask = t['TOTCORR'].mask | (t['MJD-OBS'] < SM1end.mjd).data +Nmasked(t) +``` + + 4709 masked out of 12058 for TOTCORR + + + +```python +for EMbin in [1, 2, 3, 4, 'SUM']: + t.add_column(MaskedColumn(t['GAIAMAG'] + 2.5*np.log10(t[f'TOTCORR_{EMbin}'] / t['EXPTIME']), name=f'EMZP_{EMbin}')) + t[f'TOTCORR_{EMbin}'].mask = t['TOTCORR'].mask | t[f'TOTCORR_{EMbin}'].mask | (t[f'TOTCORR_{EMbin}'] <= 0) + t[f'EMZP_{EMbin}'].mask = t[f'TOTCORR_{EMbin}'].mask + Nmasked(t, column=f'TOTCORR_{EMbin}') + Nmasked(t, column=f'EMZP_{EMbin}') + +``` + + 4712 masked out of 12058 for TOTCORR_1 + 4712 masked out of 12058 for EMZP_1 + 4709 masked out of 12058 for TOTCORR_2 + 4709 masked out of 12058 for EMZP_2 + 4710 masked out of 12058 for TOTCORR_3 + 4710 masked out of 12058 for EMZP_3 + 4710 masked out of 12058 for TOTCORR_4 + 4710 masked out of 12058 for EMZP_4 + 4709 masked out of 12058 for TOTCORR_SUM + 4709 masked out of 12058 for EMZP_SUM + + + /var/folders/3m/rvt2qsdx3nv0cv6m5rvbbgzc0000gp/T/ipykernel_46210/3329158793.py:2: RuntimeWarning: divide by zero encountered in log10 + t.add_column(MaskedColumn(t['GAIAMAG'] + 2.5*np.log10(t[f'TOTCORR_{EMbin}'] / t['EXPTIME']), name=f'EMZP_{EMbin}')) + /var/folders/3m/rvt2qsdx3nv0cv6m5rvbbgzc0000gp/T/ipykernel_46210/3329158793.py:2: RuntimeWarning: invalid value encountered in log10 + t.add_column(MaskedColumn(t['GAIAMAG'] + 2.5*np.log10(t[f'TOTCORR_{EMbin}'] / t['EXPTIME']), name=f'EMZP_{EMbin}')) + + + +```python +def calculate_ExpMeter_zero_point(t, EMbin=1, percentile=90, plot=False): + mask = t[f'EMZP_{EMbin}'].mask + EMZP = float(np.percentile(t[f'EMZP_{EMbin}'][~mask], percentile)) + # print(f"Exposure Meter Zero Point = {EMZP:.2f}") + + if plot: + plt.figure(figsize=(10,3)) + plt.title(f'Exposure Meter Zero Point (bin {EMbin}) = {EMZP:.2f}') + plt.hist(t[f'EMZP_{EMbin}'][~mask], bins=200, density=True, label='Zero Points') + plt.ylabel(f'Density (N={len(t[~mask])})') + plt.axvline(EMZP, color='k', alpha=0.5, label='Nominal Zero Point') + plt.grid() + plt.legend(loc='best') + plt.xlim(12,24) + plt.xlabel('Zero Point (mag)') + plt.show() + + return EMZP +``` + + +```python +EMZeroPoint = {} +for EMbin in [1, 2, 3, 4, 'SUM']: + EMZeroPoint[EMbin] = calculate_ExpMeter_zero_point(t, EMbin=EMbin, percentile=90, plot=True) +``` + + /Users/jwalawender/miniforge3/envs/py313/lib/python3.13/site-packages/numpy/lib/_function_base_impl.py:4786: UserWarning: Warning: 'partition' will ignore the 'mask' of the MaskedColumn. + arr.partition( + + + + +![png](output_9_1.png) + + + + + +![png](output_9_2.png) + + + + + +![png](output_9_3.png) + + + + + +![png](output_9_4.png) + + + + + +![png](output_9_5.png) + + + +### Discussion + +The distribution of individual zero point values follows roughly what one might expect: a peak at high zero point corresponding to good (cloudless) conditions. The width of that peak is probably related to the seeing distribution of these observations. The long tail of zero points at lower values correspond to cloudy nights with poor throughput. + +It is notable that the exposure meter's "bin 4" (wavelength of about 810nm) has a stronger tail to very high zero points. It is not clear what is different here, but this is the extreme edge of the exposure meter's wavelength range. + + +```python +EMZeroPoint +``` + + + + + {1: 21.296861833059204, + 2: 21.802875631774594, + 3: 21.73859924329517, + 4: 21.385171098414354, + 'SUM': 23.06521519492087} + + + +## Check for Evolution Over Time + +We bin the zero point values by day (MJD) to see if we can discern any time evolution. + + +```python +mjd = np.array([int(d) for d in t['MJD-OBS']]) + +plt.figure(figsize=(12,6)) + +for EMbin in [1, 2, 3, 4]: + mask = t[f'EMZP_{EMbin}'].mask + plt.subplot(2,2,EMbin) + color = ['b', 'c', 'g', 'r'][EMbin-1] + for j,day in enumerate(set(mjd)): + wday = (mjd == day) & ~mask + if np.sum(np.array(wday, dtype=int)) > 9: + zp = np.median(t[f'EMZP_{EMbin}'][wday]) + plt.plot(day, zp, f'{color}o', alpha=0.4, ms=4) + plt.axvspan(SM1start.mjd, SM1end.mjd, color='k', alpha=0.3, label='Servicing Mission') + plt.axvspan(SM2start.mjd, SM2end.mjd, color='k', alpha=0.3) + plt.axvspan(SM3start.mjd, SM3end.mjd, color='k', alpha=0.3) + plt.axvspan(SM4start.mjd, SM4end.mjd, color='k', alpha=0.3) + plt.ylabel(f'Zero Point (EMbin={EMbin})') + plt.legend(loc='best') + # plt.gca().set_xticklabels([]) + if EMbin in [3, 4]: + plt.xlabel('MJD') + plt.grid() + +plt.show() +``` + + /Users/jwalawender/miniforge3/envs/py313/lib/python3.13/site-packages/numpy/_core/fromnumeric.py:840: UserWarning: Warning: 'partition' will ignore the 'mask' of the MaskedColumn. + a.partition(kth, axis=axis, kind=kind, order=order) + + + + +![png](output_13_1.png) + + + +### Discussion + +There are hints of a time evolution here. Specifically there appears to be a drop in zero points after the second servicing mission. + +Note that data from before the first servicing mission is not shown as the reformatter was realigned during that servicing mission. Prior to that realignment, the fraction of light sent to the exposure meter was higher. + +# Save Results to Disk + + +```python +t.write('data/Feb2026_withEMZPs.csv', overwrite=True) +``` + + +```python +EMZPfile = Path('data/ExposureMeterZeroPoints.yaml') +if EMZPfile.exists(): EMZPfile.unlink() +with open(EMZPfile, 'w') as f: + f.write(yaml.dump(EMZeroPoint)) +``` diff --git a/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_13_1.png b/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_13_1.png new file mode 100644 index 000000000..b759c4cd2 Binary files /dev/null and b/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_13_1.png differ diff --git a/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_9_1.png b/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_9_1.png new file mode 100644 index 000000000..124bcf127 Binary files /dev/null and b/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_9_1.png differ diff --git a/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_9_2.png b/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_9_2.png new file mode 100644 index 000000000..9269c2bb0 Binary files /dev/null and b/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_9_2.png differ diff --git a/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_9_3.png b/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_9_3.png new file mode 100644 index 000000000..0021303a9 Binary files /dev/null and b/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_9_3.png differ diff --git a/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_9_4.png b/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_9_4.png new file mode 100644 index 000000000..e14a8b0dc Binary files /dev/null and b/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_9_4.png differ diff --git a/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_9_5.png b/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_9_5.png new file mode 100644 index 000000000..62ad390a8 Binary files /dev/null and b/docs/ZeroPointAnalysis/2-ExpMeterZeroPoint/output_9_5.png differ diff --git a/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/3-InstrumentZeroPoint.md b/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/3-InstrumentZeroPoint.md new file mode 100644 index 000000000..af459e3b3 --- /dev/null +++ b/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/3-InstrumentZeroPoint.md @@ -0,0 +1,256 @@ +# Determine Instrument Zero Points + +Each observation can be used as a measure of the zero point of the spectrograph from the DRP's `SNRSCnnn` FITS header values. We calulate the flux at each wavelength by squaring the SNR value. By examining the distribution of these zero point values, we can try to determine the nominal zero point -- the zero point we would achieve under nominal (i.e. good) conditions. We'll take the 90th percentile of the distribution as "nominal" for this calculation. + + +```python +from pathlib import Path +import yaml +import numpy as np +from astropy import stats +from astropy.time import Time +from astropy.table import Table, MaskedColumn +from kpf_etc.etc import kpf_photon_noise_estimate + +import matplotlib.pyplot as plt + +data_file = 'data/Feb2026_withEMZPs.csv' +t = Table.read(data_file, format='ascii.csv') +``` + + +```python +wavs = ['452', '548', '652', '747', '852'] +``` + + +```python +def Nmasked(t, wav='548'): + mask_count = int(np.sum(np.array(t[f'SNRSC{wav}'].mask, dtype=int))) + print(f"{mask_count} masked out of {len(t)}") +``` + + +```python +Nmasked(t) +``` + + 36 masked out of 12058 + + + +```python +# KPFERA 1.0 is before SM1 +SM1start = Time('2024-02-03') +SM1end = Time('2024-02-23') +# KPFERA 2.0 is between SM1 and SM2 +SM2start = Time('2024-10-31') +SM2end = Time('2024-11-20') +# KPFERA 2.5/2.6 is between SM2 and SM3 +SM3start = Time('2025-03-28') +SM3end = Time('2025-04-23') +# KPFERA 3.0 is between SM3 and SM4 +SM4start = Time('2025-08-31') +SM4end = Time('2025-10-27') +# KPFERA 4.0 is after SM4 + +KPFEra = MaskedColumn(data=['?']*len(t), mask=np.zeros(len(t), dtype=bool), name='KPFEra', dtype='a3') + +KPFEra[(t['MJD-OBS'] > 59823) & (t['MJD-OBS'] < SM1start.mjd)] = '1.0' +KPFEra[(t['MJD-OBS'] > SM1end.mjd) & (t['MJD-OBS'] < SM2start.mjd)] = '2.0' +KPFEra[(t['MJD-OBS'] > SM2end.mjd) & (t['MJD-OBS'] < SM3start.mjd)] = '2.5' +KPFEra[(t['MJD-OBS'] > SM3end.mjd) & (t['MJD-OBS'] < SM4start.mjd)] = '3.0' +KPFEra[(t['MJD-OBS'] > SM4end.mjd)] = '4.0' +KPFEra.mask = KPFEra == '?' +t.add_column(KPFEra) + +eras = ['1.0', '2.0', '2.5', '3.0', '4.0'] + +print("Era Nfiles") +print("___________") +for era in eras: + print(era, len(KPFEra[KPFEra == era])) +print(' ? ', int(np.sum(np.array(KPFEra.mask, dtype=int)))) +``` + + Era Nfiles + ___________ + 1.0 4695 + 2.0 3423 + 2.5 1588 + 3.0 1639 + 4.0 713 + ? 0 + + + +```python +for wav in wavs: + t.add_column(MaskedColumn(t['GAIAMAG'] + 2.5*np.log10(t[f'SNRSC{wav}']**2 / t['EXPTIME']), name=f'ZP_{wav}')) + t[f'SNRSC{wav}'].mask = t[f'SNRSC{wav}'].mask | (t[f'SNRSC{wav}'] <= 0) + t[f'ZP_{wav}'].mask = t[f'SNRSC{wav}'].mask + Nmasked(t, wav=wav) +``` + + 49 masked out of 12058 + 36 masked out of 12058 + 35 masked out of 12058 + 35 masked out of 12058 + 35 masked out of 12058 + + + /var/folders/3m/rvt2qsdx3nv0cv6m5rvbbgzc0000gp/T/ipykernel_46279/3421180547.py:2: RuntimeWarning: divide by zero encountered in log10 + t.add_column(MaskedColumn(t['GAIAMAG'] + 2.5*np.log10(t[f'SNRSC{wav}']**2 / t['EXPTIME']), name=f'ZP_{wav}')) + + + +```python +def calculate_instrument_zero_point(t, wav='652', percentile=90, plot=False): + mask = t[f'SNRSC{wav}'].mask + ZP = float(np.percentile(t[f'ZP_{wav}'][~mask], percentile)) + + if plot: + plt.figure(figsize=(10,3)) + plt.title(f'Instrument Zero Point ({wav} nm) = {ZP:.2f}') + plt.hist(t[f'ZP_{wav}'][~mask], bins=200, density=True, label='Zero Points') + plt.ylabel(f'Density (N={len(t[~mask])})') + plt.axvline(ZP, color='k', alpha=0.5, label='Nominal Zero Point') + plt.grid() + plt.legend(loc='best') + plt.xlim(5,17) + plt.xlabel('Zero Point (mag)') + plt.show() + + return ZP +``` + + +```python +ZeroPoint = {} +for wav in wavs: + ZeroPoint[wav] = calculate_instrument_zero_point(t, wav=wav, percentile=90, plot=True) +``` + + /Users/jwalawender/miniforge3/envs/py313/lib/python3.13/site-packages/numpy/lib/_function_base_impl.py:4786: UserWarning: Warning: 'partition' will ignore the 'mask' of the MaskedColumn. + arr.partition( + + + + +![png](output_8_1.png) + + + + + +![png](output_8_2.png) + + + + + +![png](output_8_3.png) + + + + + +![png](output_8_4.png) + + + + + +![png](output_8_5.png) + + + +### Discussion + +As with the Exposure Meter, the distribution of individual zero point values follows roughly what one might expect: a peak at high zero point corresponding to good (cloudless) conditions. The width of that peak is probably related to the seeing distribution of these observations. The long tail of zero points at lower values correspond to cloudy nights with poor throughput. + +Similar to the Exposure Meter's "bin 4" behavior, the longer wavelengths have a stronger tail to very high zero points. + + + +```python +ZeroPoint +``` + + + + + {'452': 13.664362089856091, + '548': 14.964690898633878, + '652': 15.329155098901115, + '747': 15.466755734095996, + '852': 15.073233551565435} + + + +## Check for Evolution Over Time + +We bin the zero point values by day (MJD) to see if we can discern any time evolution. For the main spectrograph, this is obscured by the fact that data were reduced with different versions of the DRP, which can perform differently at the extraction step. We'll color code each of the major DRP versions and slice the data by both day and DRP version. + + +```python +mjd = np.array([int(d) for d in t['MJD-OBS']]) + +plt.figure(figsize=(12,12)) + +for w,wav in enumerate(wavs): + plt.subplot(3,2,w+1) + plt.title(f'Zero Point Evolution at {wav} nm') + drpvs = ['v2.9.1', 'v2.10.3', 'v2.12.0rc1'] + for i,drpv in enumerate(drpvs): + color = ['r', 'g', 'b'][i] + labeled = False + for j,day in enumerate(set(mjd)): + wday = np.where((mjd == day) & (t['DRPTAGL2'] == drpv)) + if len(wday[0]) > 0: + day_zp = np.median(t[f'ZP_{wav}'][wday]) + if not labeled: + plt.plot(day, day_zp, f'{color}o', alpha=0.3, label=f'{drpvs[i]}') + labeled = True + else: + plt.plot(day, day_zp, f'{color}o', alpha=0.3) + plt.axvspan(SM1start.mjd, SM1end.mjd, color='k', alpha=0.3, label='Servicing Mission') + plt.axvspan(SM2start.mjd, SM2end.mjd, color='k', alpha=0.3) + plt.axvspan(SM3start.mjd, SM3end.mjd, color='k', alpha=0.3) + plt.axvspan(SM4start.mjd, SM4end.mjd, color='k', alpha=0.3) + if w in [3, 4]: plt.xlabel('MJD') + plt.ylabel('Zero Point') + plt.grid() + plt.legend(loc='best') + +plt.show() +``` + + /Users/jwalawender/miniforge3/envs/py313/lib/python3.13/site-packages/numpy/_core/fromnumeric.py:840: UserWarning: Warning: 'partition' will ignore the 'mask' of the MaskedColumn. + a.partition(kth, axis=axis, kind=kind, order=order) + + + + +![png](output_12_1.png) + + + +### Discussion + +Because the DRP version occurance roughly tracks the KPF Era, it is hard to be conclusive about time evolution, but there appears to be a general downward trend. When comparing KPF Era 2.0 and 4.0 which both dominantly use v2.12.0rc1, there does appears to be a downward step in zero points. + +# Save Results to Disk + + +```python +t.write('data/Feb2026_withInstZPs.csv', overwrite=True) +``` + + +```python +ZPfile = Path('data/InstrumentZeroPoints.yaml') +if ZPfile.exists(): ZPfile.unlink() +with open(ZPfile, 'w') as f: + f.write(yaml.dump(ZeroPoint)) +``` diff --git a/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_12_1.png b/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_12_1.png new file mode 100644 index 000000000..71983b094 Binary files /dev/null and b/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_12_1.png differ diff --git a/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_8_1.png b/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_8_1.png new file mode 100644 index 000000000..0b04801d0 Binary files /dev/null and b/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_8_1.png differ diff --git a/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_8_2.png b/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_8_2.png new file mode 100644 index 000000000..7c2ad7e27 Binary files /dev/null and b/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_8_2.png differ diff --git a/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_8_3.png b/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_8_3.png new file mode 100644 index 000000000..276360906 Binary files /dev/null and b/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_8_3.png differ diff --git a/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_8_4.png b/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_8_4.png new file mode 100644 index 000000000..50df1a8cc Binary files /dev/null and b/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_8_4.png differ diff --git a/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_8_5.png b/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_8_5.png new file mode 100644 index 000000000..08ede5f30 Binary files /dev/null and b/docs/ZeroPointAnalysis/3-InstrumentZeroPoint/output_8_5.png differ diff --git a/docs/ZeroPointAnalysis/4-BestExpMeterBin/4-BestExpMeterBin.md b/docs/ZeroPointAnalysis/4-BestExpMeterBin/4-BestExpMeterBin.md new file mode 100644 index 000000000..eb1058f96 --- /dev/null +++ b/docs/ZeroPointAnalysis/4-BestExpMeterBin/4-BestExpMeterBin.md @@ -0,0 +1,189 @@ +# Determine the Most Predictive Exposure Meter Bin for Each Science Wavelength + +One use case for the exposure meter is that it can be used to terminate the exposure when a certain signal level is reached. In this mode, we want the exposure meter to predict the signal at one of the DRP's chosen wavelengths. Which exposure meter wavelength bin is the best predictor for each of the DRP wavelengths? To determine this we will look at the spread in zero point difference values and choose the exposure meter bin with the narrowest spead for each spectrograph wavelength. + +Naievely we would expect the correspondance to follow the wavelengths and that is what we get from this analysis, but it is reassuring to see that confirmed. + + +```python +from pathlib import Path +import yaml +import numpy as np +from astropy import stats +from astropy.time import Time +from astropy.table import Table, MaskedColumn +from kpf_etc.etc import kpf_photon_noise_estimate + +import matplotlib.pyplot as plt + +data_file = 'data/Feb2026_withInstZPs.csv' +t = Table.read(data_file, format='ascii.csv') +``` + + +```python +wavs = ['452', '548', '652', '747', '852'] +EMbins = [1, 2, 3, 4] +``` + +# Find the "Best" Exposure Meter Bin for Each Science Wavelength + +Look at the spread in zero point differences and find the Exposure Meter Bin with the least scatter in zero poitn differences and use that at the "best" exposure meter bin to use for the science wavelength under test. + + +```python +def find_best_ExpMeterBin(t, wav, plot=True): + EMwav = {1: ' 498nm', + 2: ' 604nm', + 3: ' 710nm', + 4: ' 817nm', + } + + if plot: plt.figure(figsize=(12,2)) + comparison_stddev = [] + comparison_median = [] + for EMbin in EMbins: + ZPdiff = t[f'ZP_{wav}'] - t[f'EMZP_{EMbin}'] + mask = t[f'ZP_{wav}'].mask | t[f'EMZP_{EMbin}'].mask + mean, median, stddev = stats.sigma_clipped_stats(ZPdiff[~mask]) + comparison_stddev.append(float(stddev)) + comparison_median.append(float(median)) + + plt.subplot(1,4,EMbin) + plt.title(f'{wav}nm-{EMbin}: {median:.2f}+/-{stddev:.3f}') + bins = np.arange(median-10*stddev, median+10*stddev, 0.01) + plt.hist(ZPdiff[~mask], bins=bins, color='b', alpha=0.4, + density=True, log=False) + + if EMbin == 1: plt.ylabel(f'Density (N={len(ZPdiff)})') + plt.gca().set_xticks([]) + plt.xlabel(f'') + + plt.show() + best_EMbin = EMbins[np.argmin(comparison_stddev)] + return best_EMbin, EMwav[best_EMbin].strip() +``` + + +```python +best_EMbin = {} +for wav in wavs: + EMbin, EMwav = find_best_ExpMeterBin(t, wav, plot=True) + best_EMbin[wav] = EMbin +``` + + + +![png](output_5_0.png) + + + + + +![png](output_5_1.png) + + + + + +![png](output_5_2.png) + + + + + +![png](output_5_3.png) + + + + + +![png](output_5_4.png) + + + + +```python +best_EMbin +``` + + + + + {'452': 1, '548': 2, '652': 3, '747': 4, '852': 4} + + + +# Save Results to Disk + + +```python +EMbinfile = Path('data/ExpMeterBins.yaml') +if EMbinfile.exists(): EMbinfile.unlink() +with open(EMbinfile, 'w') as f: + f.write(yaml.dump(best_EMbin)) +``` + +# Look at Conversion Factors + +This is doing the same examination as above, but in linear space instead of log space (magnitudes). We're doing this purely as a verification step to make sure we get the same result. + + +```python +for wav in wavs: + for EMbin in EMbins: + t.add_column(MaskedColumn(t[f'TOTCORR_{EMbin}']/t[f'SNRSC{wav}']**2), name=f'CV_{wav}_{EMbin}') + t[f'CV_{wav}_{EMbin}'].mask = t[f'TOTCORR_{EMbin}'].mask | t[f'SNRSC{wav}'].mask | (t[f'TOTCORR_{EMbin}'] <= 0) | (t[f'SNRSC{wav}'] <= 0) + +``` + + +```python +conversion = {} + +plt.figure(figsize=(14,12)) +for i,wav in enumerate(wavs): + for EMbin in EMbins: + best = best_EMbin[wav] == EMbin + hcolor = {True: 'g', False: 'b'}[best] + plt.subplot(5,4,i*4+EMbin) + conversions = t[f'CV_{wav}_{EMbin}'][~t[f'CV_{wav}_{EMbin}'].mask] + Nc = len(conversions) + mean, median, stdev = stats.sigma_clipped_stats(conversions) + bins = np.arange(median-7*stdev, median+7*stdev, np.ceil(stdev)/5) + n, b, _ = plt.hist(conversions, bins=bins, color=hcolor, density=True) + wmode = np.argmax(n) + mode = (b[wmode]+b[wmode+1])/2 + if best: + conversion[f"{wav}"] = float(median) + title = f"{wav}-{EMbin}: {median:.0f}+/-{stdev:.0f}" + plt.title(title) + plt.axvline(median, color='k', alpha=0.3, label='median') + plt.axvline(mode, color='r', alpha=0.6, label='mode') + # plt.legend(loc='best') + plt.gca().set_xticks([]) +plt.show() +``` + + + +![png](output_11_0.png) + + + +### Discussion + +From this we get the conversion rate between Exposure Meter ADU and spectrograph detected photons which we can compare against the previously calculated zero point difference and see that they are consistent. + + +```python +for wav in wavs: + print(f"{wav}nm: {conversion[wav]:4.0f} EMADU/photon = {-2.5*np.log10(conversion[wav]):.2f} mag zero point difference") +``` + + 452nm: 1193 EMADU/photon = -7.69 mag zero point difference + 548nm: 583 EMADU/photon = -6.91 mag zero point difference + 652nm: 406 EMADU/photon = -6.52 mag zero point difference + 747nm: 260 EMADU/photon = -6.04 mag zero point difference + 852nm: 392 EMADU/photon = -6.48 mag zero point difference + diff --git a/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_11_0.png b/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_11_0.png new file mode 100644 index 000000000..cfa4c3258 Binary files /dev/null and b/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_11_0.png differ diff --git a/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_5_0.png b/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_5_0.png new file mode 100644 index 000000000..c0b71de5d Binary files /dev/null and b/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_5_0.png differ diff --git a/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_5_1.png b/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_5_1.png new file mode 100644 index 000000000..30d2f2632 Binary files /dev/null and b/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_5_1.png differ diff --git a/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_5_2.png b/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_5_2.png new file mode 100644 index 000000000..4de69d336 Binary files /dev/null and b/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_5_2.png differ diff --git a/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_5_3.png b/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_5_3.png new file mode 100644 index 000000000..3148af26d Binary files /dev/null and b/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_5_3.png differ diff --git a/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_5_4.png b/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_5_4.png new file mode 100644 index 000000000..05d643a79 Binary files /dev/null and b/docs/ZeroPointAnalysis/4-BestExpMeterBin/output_5_4.png differ diff --git a/docs/ZeroPointAnalysis/5-ETCs/5-ETCs.md b/docs/ZeroPointAnalysis/5-ETCs/5-ETCs.md new file mode 100644 index 000000000..ec8b1f7c3 --- /dev/null +++ b/docs/ZeroPointAnalysis/5-ETCs/5-ETCs.md @@ -0,0 +1,424 @@ +# Examine Slowdown Factors + +The nominal zero points we have determined are effectively simple ETCs for each wavelength as, by definition, the zero point is the magnitude at which you will get a flux of 1 per second of whatever flux units you used to calculate the zero point. In our case, this is detected photons per pixel for the spectrograph and total ADU in the wavelength bin for the Exposure Meter. + +Because these zero points represent nominal conditions, we can use the comparison of the ETC predictions to the actual measured flux to measure the slowdown factor which is of interest to KPF-CC scheduling. + + +```python +from pathlib import Path +import yaml +import numpy as np +from astropy import stats +from astropy.time import Time +from astropy.table import Table, MaskedColumn +from kpf_etc.etc import kpf_photon_noise_estimate + +import matplotlib.pyplot as plt + +data_file = 'data/Feb2026_withInstZPs.csv' +t = Table.read(data_file, format='ascii.csv') +``` + + +```python +EMZPfile = Path('data/ExposureMeterZeroPoints.yaml') +with open(EMZPfile, 'r') as f: + EMZeroPoint = yaml.safe_load(f) +EMZeroPoint +``` + + + + + {1: 21.296861833059204, + 2: 21.802875631774594, + 3: 21.73859924329517, + 4: 21.385171098414354, + 'SUM': 23.06521519492087} + + + + +```python +ZPfile = Path('data/InstrumentZeroPoints.yaml') +with open(ZPfile, 'r') as f: + ZeroPoint = yaml.safe_load(f) +ZeroPoint +``` + + + + + {'452': 13.664362089856091, + '548': 14.964690898633878, + '652': 15.329155098901115, + '747': 15.466755734095996, + '852': 15.073233551565435} + + + + +```python +EMbinfile = Path('data/ExpMeterBins.yaml') +with open(EMbinfile, 'r') as f: + best_EMbin = yaml.safe_load(f) +best_EMbin +``` + + + + + {'452': 1, '548': 2, '652': 3, '747': 4, '852': 4} + + + + +```python +wavs = ZeroPoint.keys() +EMbins = EMZeroPoint.keys() +``` + +# Use "Nominal" Zero Point Value as an ETC + + +```python +def ZPETC(Gmag, exptime, wav='652', EMbin=2): + assert wav in ZeroPoint.keys() + assert EMbin in EMZeroPoint.keys() + photons = float(exptime*10**((Gmag - ZeroPoint[wav])/-2.5)) + EM_ADU = float(exptime*10**((Gmag - EMZeroPoint[EMbin])/-2.5)) + return photons, EM_ADU + + +def EMThreshold(photons, wav='652', EMbin=2): + assert wav in ZeroPoint.keys() + assert EMbin in EMZeroPoint.keys() + ZPdiff = ZeroPoint[wav] - EMZeroPoint[EMbin] + EM_ADU = photons*10**(-ZPdiff/2.5) + return EM_ADU +``` + + +```python +# Double Check Consistency +photons, EMADU = ZPETC(10, 60, wav='747', EMbin=2) +EMADU2 = EMThreshold(photons, wav='747', EMbin=2) +EMADU2/EMADU +``` + + + + + 1.0 + + + +## Add Zero Point ETC results to table for both instrument and exposure meter + + +```python +for wav in wavs: + t.add_column(MaskedColumn(t['ELAPSED']*10**((t['GAIAMAG'] - ZeroPoint[wav])/-2.5), name=f'ZP-etc-{wav}')) + t[f'ZP-etc-{wav}'].mask = t[f'ZP-etc-{wav}'].mask | (t[f'ZP-etc-{wav}'] <= 0) + slowdown = t[f'ZP-etc-{wav}']/t[f'SNRSC{wav}']**2 + t.add_column(MaskedColumn(slowdown, name=f'SD-ZP-etc-{wav}')) + t[f'SD-ZP-etc-{wav}'].mask = t[f'SD-ZP-etc-{wav}'].mask | t[f'ZP-etc-{wav}'].mask | (t[f'SD-ZP-etc-{wav}'] > 10) +``` + + +```python +for EMbin in EMbins: + t.add_column(MaskedColumn(t['ELAPSED']*10**((t['GAIAMAG'] - EMZeroPoint[EMbin])/-2.5), name=f'ZP-etc-EM{EMbin}')) + t[f'ZP-etc-EM{EMbin}'].mask = t[f'ZP-etc-EM{EMbin}'].mask | (t[f'ZP-etc-EM{EMbin}'] <= 0) + slowdown = t[f'ZP-etc-EM{EMbin}']/t[f'TOTCORR_{EMbin}'] + t.add_column(MaskedColumn(slowdown, name=f'SD-ZP-etc-EM{EMbin}')) + t[f'SD-ZP-etc-EM{EMbin}'].mask = t[f'SD-ZP-etc-EM{EMbin}'].mask | t[f'ZP-etc-EM{EMbin}'].mask | (t[f'SD-ZP-etc-EM{EMbin}'] > 10) +``` + +## Add in results of formal KPF-etc + + +```python +def read_GminusV_table(): + '''Table of data from: + https://www.pas.rochester.edu/~emamajek/EEM_dwarf_UBVIJHK_colors_Teff.txt + ''' + table_file = Path('data/EEM_dwarf_UBVIJHK_colors_Teff.txt') + if table_file.exists() is False: + return 0 + t = Table.read(table_file, format='ascii') + filtered = t[t['G-V'] != '...'] + return filtered + +filtered = read_GminusV_table() + +def get_GminusV(Teff): + Teff_diff = abs(filtered['Teff'] - Teff) + ind = np.argmin(Teff_diff) + GminusV = float(filtered[ind]['G-V']) + return GminusV + +def Vmag(Gmag, Teff): + return Gmag - get_GminusV(Teff) +``` + + +```python +# Add Vmag as column +vmag_data = [Vmag(row['GAIAMAG'], row['TARGTEFF']) for row in t] +t.add_column(MaskedColumn(vmag_data, name=f'VMAG')) +``` + + +```python +# Add ETC results as columns +order = {'452': 2, + '548': 25, + '652': 43, + '747': 55, + '852': 65} +ETCflux = {'452': [], + '548': [], + '652': [], + '747': [], + '852': []} +ETC_SD = {'452': [], + '548': [], + '652': [], + '747': [], + '852': []} + +ETCvalid = (t['TARGTEFF'] >= 2700) & (t['TARGTEFF'] <= 6600) & (t['ELAPSED'] > 10) & (t['ELAPSED'] <= 3600) +for i,row in enumerate(t): + # KPF-etc + if ETCvalid[i] == True: + try: + a, b, snr_ord, c = kpf_photon_noise_estimate(row['TARGTEFF'], row['VMAG'], row['ELAPSED'], quiet=True) + for wav in wavs: + predicted_flux = float(snr_ord[order[wav]]**2) + ETCflux[wav].append(predicted_flux) + slowdown = predicted_flux/float(row[f'SNRSC{wav}']**2) + ETC_SD[wav].append(slowdown) + except Exception as e: + print(e, row['TARGTEFF'], row['VMAG'], row['ELAPSED']) + for wav in wavs: + ETCflux[wav].append(0) + ETC_SD[wav].append(0) + else: + for wav in wavs: + ETCflux[wav].append(0) + ETC_SD[wav].append(0) + +for wav in wavs: + t.add_column(MaskedColumn(ETCflux[wav], name=f'KPF-etc-{wav}')) + t[f'KPF-etc-{wav}'].mask = (t[f'KPF-etc-{wav}'] <= 0) + t.add_column(MaskedColumn(ETC_SD[wav], name=f'SD-KPF-etc-{wav}')) + t[f'SD-KPF-etc-{wav}'].mask = (t[f'SD-KPF-etc-{wav}'] <= 0) | (t[f'SD-KPF-etc-{wav}'] > 10) + +``` + + /var/folders/3m/rvt2qsdx3nv0cv6m5rvbbgzc0000gp/T/ipykernel_69776/848789479.py:27: UserWarning: Warning: converting a masked element to nan. + slowdown = predicted_flux/float(row[f'SNRSC{wav}']**2) + + +# Slowdown Factor + +How well does the nominal flux estimate from our Zero Point based ETC predict the actual/measured flux in the data set? To examine this, we look at the slowdown factor which is the ratio of the flux expected versus the flux measured. The slowdown can be calculated for the instrument flux and for the Exposure Meter flux, so we want to see how well the Exposure Meter slowdown predicts the spectrograph slowdown. + +During night observations, the Exposure Meter slowdown factor could be calculated in real time, but the spectrograph slowdown factor will only be available once the DRP has run and provided the `SNRSC*` keyword values. + +We can use either the Zero Point ETC or the formal KPF-etc (when the target is within its input ranges) to calculate a slowdown factor for the spectrograph, so we will examine both. + + +```python +def plot_one_slowdown(slowdowns, bins=np.arange(0,10+0.05,0.05), color='b', label='Spectrograph'): + plt.hist(slowdowns, bins=bins, density=True, color=color, label=label) + n, binedges = np.histogram(slowdowns, bins=[0,1,2,4,1e6]) + d = [float(val/np.sum(n)) for val in n] + plt.axvspan(0, 1, color='k', alpha=0.01, label=f'band0 {d[0]:.1%}') + plt.axvspan(1, 2, color='g', alpha=0.2, label=f'band1 {d[1]:.1%}') + plt.axvspan(2, 4, color='y', alpha=0.2, label=f'band2 {d[2]:.1%}') + plt.axvspan(4, 10, color='r', alpha=0.1, label=f'band3 {d[3]:.1%}') + plt.grid() + plt.legend(loc='best') + plt.ylabel(f'Density') + return d + + +def plot_slowdowns(t, wav): + EMbin = best_EMbin[wav] + bins = np.arange(0,10+0.05,0.05) + + plt.figure(figsize=(10,6)) + + plt.subplot(3,1,1) + ZPETCslowdowns = t[f'SD-ZP-etc-{wav}'][~t[f'SD-ZP-etc-{wav}'].mask] + plt.title(f'Instrument Slowdowns Calculated at {wav} nm') + d = plot_one_slowdown(ZPETCslowdowns, bins=bins, color='b', label='Spectrograph') + + plt.subplot(3,1,2) + EM_ZPETCslowdowns = t[f'SD-ZP-etc-EM{EMbin}'][~t[f'SD-ZP-etc-EM{EMbin}'].mask] + plot_one_slowdown(EM_ZPETCslowdowns, bins=bins, color='g', label='ExpMeter') + + plt.subplot(3,1,3) + KPFETCslowdowns = t[f'SD-KPF-etc-{wav}'][~t[f'SD-KPF-etc-{wav}'].mask] + plot_one_slowdown(KPFETCslowdowns, bins=bins, color='c', label='KPF-etc') + plt.xlabel('Slowdown Factor') + + plt.show() + return d +``` + + +```python +distribution = {} +for wav in wavs: + distribution[wav] = plot_slowdowns(t, wav) +``` + + + +![png](output_18_0.png) + + + + + +![png](output_18_1.png) + + + + + +![png](output_18_2.png) + + + + + +![png](output_18_3.png) + + + + + +![png](output_18_4.png) + + + +### Discussion: Spectrograph and Exposure Meter Slowdowns + +We see that the slowdown factors for both the instrument and the exposure meter are typically >1. This is to be expected as our nominal zero points are the 90th percentile of good data, so the Zero Point ETC assumes very good conditions, thus most data are taken in somewhat worse conditions. The distribution of slowdowns is what we'd expect given the distribution of Zero Points we measured before. + +We have highlighted the weather bands as defined by their slowdown rate and put the percentage of observations in each band in the legend. The majority of observations take place in band1 as we would expect. The "band0" designation is for observations with a slowdown of <1 which should probably be considered part of band1, but is nominally out of the definied range. For the spectrograph and exposure meter, the occurance rate of band0 is ~10% which is reasonable based our use of the 90th percentile to define "nominal" which would correspond to a slowdown factor of 1.0. + +Comparing the distribution of exposure meter slowdowns to instrument slow downs, we see that the exposure meter is typically a bit optimistic relative to the instrument by 10-15%, so the actual slowdown is a bit worse than one estimated in real time by the Exposure Meter. + +### Discussion: KPF-etc Calculated Slowdowns + +For all wavelengths other than 452nm, under good conditions, the formal KPF-etc underestimates the flux relative to what is actually observed leading to a "slowdown" of less than 1, putting most of the observations in "band0". + +## How Much Time is Spent in Each Weather Band? + +The above distribution of weather bands is a count of exposures, not a total time. To get that we need to sum up the elapsed exposure time in each band. + + +```python +def time_in_each_band(wav='652', etc='ZP-etc-'): + band0 = t[(t[f'SD-{etc}{wav}'] < 1)] + band1 = t[(t[f'SD-{etc}{wav}'] >= 1) & (t[f'SD-{etc}{wav}'] < 2)] + band2 = t[(t[f'SD-{etc}{wav}'] >= 2) & (t[f'SD-{etc}{wav}'] < 4)] + band3 = t[(t[f'SD-{etc}{wav}'] >= 4)] + elapsed0 = float(np.sum(band0[~band0[f'SD-{etc}{wav}'].mask]['ELAPSED'])) + elapsed1 = float(np.sum(band1[~band1[f'SD-{etc}{wav}'].mask]['ELAPSED'])) + elapsed2 = float(np.sum(band2[~band2[f'SD-{etc}{wav}'].mask]['ELAPSED'])) + elapsed3 = float(np.sum(band3[~band3[f'SD-{etc}{wav}'].mask]['ELAPSED'])) + total_elapsed = elapsed0+elapsed1+elapsed2+elapsed3 + # print(f"Time in Each Band at {wav}") + # print(f" band0: {elapsed0/total_elapsed:.1%}") + # print(f" band1: {elapsed1/total_elapsed:.1%}") + # print(f" band2: {elapsed2/total_elapsed:.1%}") + # print(f" band3: {elapsed3/total_elapsed:.1%}") + return [elapsed0/total_elapsed, elapsed1/total_elapsed, elapsed2/total_elapsed, elapsed3/total_elapsed] +``` + + +```python +duration = {} +for wav in wavs: + duration[wav] = time_in_each_band(wav=wav) +``` + + +```python +plt.figure(figsize=(8,6)) + +plt.subplot(2,1,1) +for i,wav in enumerate(wavs): + c = ['b', 'c', 'g', 'r', 'k'][i] + plt.bar(np.array([0, 1, 2, 3])+0.15*(i-2), distribution[wav], color=c, alpha=0.3, width=0.15, label=f"{wav}nm") +plt.gca().set_xticks([0, 1, 2, 3]) +plt.gca().set_xticklabels(['band0', 'band1', 'band2', 'band3']) +plt.grid() +plt.legend(loc='best') +plt.ylabel('Fraction of Exposures') + +plt.subplot(2,1,2) +for i,wav in enumerate(wavs): + c = ['b', 'c', 'g', 'r', 'k'][i] + plt.bar(np.array([0, 1, 2, 3])+0.15*(i-2), duration[wav], color=c, alpha=0.3, width=0.15, label=f"{wav}nm") +plt.gca().set_xticks([0, 1, 2, 3]) +plt.gca().set_xticklabels(['band0', 'band1', 'band2', 'band3']) +plt.grid() +plt.legend(loc='best') +plt.ylabel('Fraction of Time') + + +plt.show() +``` + + + +![png](output_23_0.png) + + + +# Look at Ratio of Actual Slowdown (from Spectrograph ZPETC) versus Exposure Meter + + +```python +plt.figure(figsize=(10,8)) +for i,wav in enumerate(wavs): + plt.subplot(5,1,i+1) + EMbin = best_EMbin[wav] + if i==0: plt.title(f"Ratio of actual slowdown to exposure meter slowdown") + ratio = t[f'SD-ZP-etc-{wav}']/t[f'SD-ZP-etc-EM{EMbin}'] + mask = t[f'SD-ZP-etc-{wav}'].mask | t[f'SD-ZP-etc-EM{EMbin}'].mask + bins=np.arange(0,max(ratio[~mask]),0.02) + plt.hist(ratio[~mask], bins=bins, density=True, label=f'{wav}nm') + plt.axvline(1.0, color='k') + plt.legend(loc='best') + plt.xlim(0.5,2.5) + plt.ylabel('Density') + plt.grid() +plt.xlabel('Slowdown Ratio: ETC Slowdown / ExpMeter Slowdown') +plt.show() +``` + + + +![png](output_25_0.png) + + + +### Discussion: Exposure Meter Slowdowns + +Comparing exposure meter slowdown measures with the spectrograph, we can see a bias of ~5-15% toward the exposure meter underestimating the slowdown. The real slowdown is larger than what you would get from the exposure meter at the end of the exposure. + +# Save Results to Disk + + +```python +t.write('data/Feb2026_withETCs.csv', overwrite=True) +``` diff --git a/docs/ZeroPointAnalysis/5-ETCs/output_18_0.png b/docs/ZeroPointAnalysis/5-ETCs/output_18_0.png new file mode 100644 index 000000000..ddd8ca249 Binary files /dev/null and b/docs/ZeroPointAnalysis/5-ETCs/output_18_0.png differ diff --git a/docs/ZeroPointAnalysis/5-ETCs/output_18_1.png b/docs/ZeroPointAnalysis/5-ETCs/output_18_1.png new file mode 100644 index 000000000..e801d9c49 Binary files /dev/null and b/docs/ZeroPointAnalysis/5-ETCs/output_18_1.png differ diff --git a/docs/ZeroPointAnalysis/5-ETCs/output_18_2.png b/docs/ZeroPointAnalysis/5-ETCs/output_18_2.png new file mode 100644 index 000000000..af56bdbb2 Binary files /dev/null and b/docs/ZeroPointAnalysis/5-ETCs/output_18_2.png differ diff --git a/docs/ZeroPointAnalysis/5-ETCs/output_18_3.png b/docs/ZeroPointAnalysis/5-ETCs/output_18_3.png new file mode 100644 index 000000000..f2a5a2021 Binary files /dev/null and b/docs/ZeroPointAnalysis/5-ETCs/output_18_3.png differ diff --git a/docs/ZeroPointAnalysis/5-ETCs/output_18_4.png b/docs/ZeroPointAnalysis/5-ETCs/output_18_4.png new file mode 100644 index 000000000..9f77cd6df Binary files /dev/null and b/docs/ZeroPointAnalysis/5-ETCs/output_18_4.png differ diff --git a/docs/ZeroPointAnalysis/5-ETCs/output_23_0.png b/docs/ZeroPointAnalysis/5-ETCs/output_23_0.png new file mode 100644 index 000000000..7f6ca0520 Binary files /dev/null and b/docs/ZeroPointAnalysis/5-ETCs/output_23_0.png differ diff --git a/docs/ZeroPointAnalysis/5-ETCs/output_25_0.png b/docs/ZeroPointAnalysis/5-ETCs/output_25_0.png new file mode 100644 index 000000000..25ccbf3e9 Binary files /dev/null and b/docs/ZeroPointAnalysis/5-ETCs/output_25_0.png differ diff --git a/docs/announcements/2026-02-01_26A-status.md b/docs/announcements/2026-02-01_26A-status.md new file mode 100644 index 000000000..0fff1e582 --- /dev/null +++ b/docs/announcements/2026-02-01_26A-status.md @@ -0,0 +1,23 @@ +# Semester 26A Stability Announcement + +Precision radial velocity measurements which are comparable across long time scales (months to years) require a stable, well calibrated instrument. KPF has had several problems in the past which have impacted stability and calibrations and these will likely impact data taken in the 26A semester. + +We discuss a few details below, but the high level summary is that we currently do not have confidence in the instrument sub-systems to maintain temperature stability and provide reliable calibrations. As a result, users of KPF who have science goals which require long term stability of the instrument should take this into account when choosing which science programs to execute with their time in 26A. + +In addition, the KPF DRP is not performing well on faint object extractions. Somewhere around Gmag of 15-16 is where the performance degrades significantly, so we advise users to avoid faint stars if they need immediate results. We do expect the DRP to handle this case properly in the long term, so data taken now will likely be useful in the future, but the results coming out at the moment will significantly underperform the ETC. + +## Temperature Stability + +The most impactful stability factor for KPF has been the detector temperatures. In order for RV measurements at different times to be compared, the system must not change temperature significantly during that time span as the hysteresis induced by a change and return to temperature results in a zero point change for the RV measurements. The goal is for the system to be stable at the ~1 mK level, while changes of ~1 K are almost certain to break the RV zero point. + +The Green and Red CCDs in the main spectrometer are likely to experience thermal events of a few degrees C (or more) above the -100 C detector setpoints lasting a few hours. While the number and timing of these are difficult to predict, it is likely that KPF will not maintain long-term RV stability over 2026A. Thus, time series RV measurements over long periods (weeks or months) are likely to be compromised by changing RV zero points at the ~10 m/s level. Experience has shown that these zero-point changes are difficult/impossible to calibrate out and are different for each star observed. Science projects that rely on a time series of KPF RVs or spectra over a short timescale (a night to a few nights) are unlikely to be affected by the warmups (which would have to occur during the observing sequence to be a problem). Users who have science goals which require high precision (a few m/s or better) on long time scales should take this into account when choosing which science programs to execute with their time in 26A. + +The WMKO and KPF Build Teams are currently planning for an additional servicing mission to address the cooling problems, however that will take time to plan and implement. + +## Calibration Reliability + +In addition to stability, the instrument also requires frequent, high precision calibrations. The most fundamental calibrator for KPF is the Laser Frequency Comb (LFC). The LFC has had periods of significant unreliability and the wavelength coverage for the LFC has been inconsistent. This has resulted in long periods of poor calibrations which requires bootstrapping the wavelength solution from other calibrators or over longer time periods which impacts RV measurement precision. + +## Detector Noise + +In addition to the stability concerns, KPF has also faced challenges with detector noise. The elevated read noise present since early 2024 has been partially mitigated. Currently, the Red detector is performing at a nominal level (4.3 electrons read noise) and although the Green detector noise has been significantly reduced from the 25-35 electrons seen in the past, it remains elevated at about 10 electrons. An intervention to address this is being planned. diff --git a/docs/announcements/2026-04-06_tubo_failure.md b/docs/announcements/2026-04-06_tubo_failure.md new file mode 100644 index 000000000..d6f5a771f --- /dev/null +++ b/docs/announcements/2026-04-06_tubo_failure.md @@ -0,0 +1,3 @@ +# April 2026 Turbo Pump Failure + +In early April, a serious incident with the Keck Planet Finder (KPF) occurred. A vacuum turbo pump that maintains KPF chamber vacuum catastrophically failed, injecting dust and some debris into the chamber housing all of the optical elements. While an initial examination indicates no obvious significant damage to the optical components, a much more rigorous inspection must be made before determining how to clean the surfaces, many of which are extremely delicate. Additionally, a full failure review analysis must be performed before KPF can be repaired and go back into science operations. At this time, we cannot provide an estimate for the return to operations of KPF, but it is a recovery likely to take several months. diff --git a/docs/buildingOBs.md b/docs/buildingOBs.md index 02a4e3dba..803cad01e 100644 --- a/docs/buildingOBs.md +++ b/docs/buildingOBs.md @@ -10,18 +10,18 @@ Observers can create OBs in 3 ways: The data in an OB can be divided in to three categories: -**Target**: The OB will contain information about the target beyond what is in a typical Keck Star List entry in order to flow that information to the FITS header and the data reduction pipeline (DRP). The target section is only needed if the OB has observations (i.e. it is not purely a calibration OB). Here is a description of all [Target Properties](../TargetProperties). +**Target**: The OB will contain information about the target beyond what is in a typical Keck Star List entry in order to flow that information to the FITS header and the data reduction pipeline (DRP). The target section is only needed if the OB has observations (i.e. it is not purely a calibration OB). Here is a description of all [Target Properties](TargetProperties.md). -**Calibrations**: An OB can contain calibrations, these are not typically used by the observer (slewcals are handled separately). Here is a description of all [Calibration Properties](../CalibrationProperties). The Calibrations section of an ON is a list of Calibration entries. +**Calibrations**: An OB can contain calibrations, these are not typically used by the observer (slewcals are handled separately). Here is a description of all [Calibration Properties](CalibrationProperties.md). The Calibrations section of an ON is a list of Calibration entries. -**Observations**: Finally, the OB will contain a list of observations to be made of the target. For typical KPF observers, this will only have one entry, but multiple entries are supported. Each entry describes a set of exposures on the target and contains the information on how those exposures should be executed. Here is a description of all [Observation Properties](../ObservationProperties). The Observations section of an ON is a list of Observation entries. +**Observations**: Finally, the OB will contain a list of observations to be made of the target. For typical KPF observers, this will only have one entry, but multiple entries are supported. Each entry describes a set of exposures on the target and contains the information on how those exposures should be executed. Here is a description of all [Observation Properties](ObservationProperties.md). The Observations section of an ON is a list of Observation entries. Note that not all properties are needed in every case. For example, an observation with `ExpMeterMode: 'monitor'` will not need values for `ExpMeterBin` and `ExpMeterThreshold`. ## Example On Sky Science OB -This is an example of what the text file form of an OB might look like. The file is a `yaml` format which resolves in to a python dict with keys for "Target", "Calibrations" and "Observations" (not all are required). The "Target" entry is a dict with the various [Target Properties](../TargetProperties). The "Calibrations" entry (if present) is a **list** of dictionaries, each with the various [Calibration Properties](../CalibrationProperties). Similarly, the "Observations" entry is a **list** of dictionaries, each with the various [Observation Properties](../ObservationProperties). +This is an example of what the text file form of an OB might look like. The file is a `yaml` format which resolves in to a python dict with keys for "Target", "Calibrations" and "Observations" (not all are required). The "Target" entry is a dict with the various [Target Properties](TargetProperties.md). The "Calibrations" entry (if present) is a **list** of dictionaries, each with the various [Calibration Properties](CalibrationProperties.md). Similarly, the "Observations" entry is a **list** of dictionaries, each with the various [Observation Properties](ObservationProperties.md). The example below has a Target, no Calibrations, and a single Observaton: diff --git a/docs/sensitivity.md b/docs/sensitivity.md index 536eb1b9c..1a2a7f9e1 100644 --- a/docs/sensitivity.md +++ b/docs/sensitivity.md @@ -9,3 +9,13 @@ We use J-band magnitude as a rough guide for the guider sensitivity, but it is i We believe this performance can be improved substantially by using sky subtraction. This is possible currently, but is a slow manual process to configure. On the second test target above (J=16.7, V=17.6), we got reasonable signal using 0.25 FPS (4 second exposures) while using sky subtraction. We are planning to script this process to make it easier and to integrate sky subtraction in to the OA's GUI for controlling the tip tilt, but the timeline for that is still TBD. If you need to acquire very faint targets (J > 16), please reach out to your SA ahead of time to discuss strategies. + +# Zero Point Analysis + +From a collection of anonymized DRP products from the first few years of observations, we have used the DRP's reported signal to noise as a probe of KPF's sensitivity. Specifically, we have measured the photometric zero point of the instrument at various wavelengths. The 5 notebooks below explore different aspects of the instrument and operations. + +- [Examine Properties of Data Set](ZeroPointAnalysis/1-PrepareDataFile/1-PrepareDataFile.md) +- [Find Zero Point of Exposure Meter](ZeroPointAnalysis/2-ExpMeterZeroPoint/2-ExpMeterZeroPoint.md) +- [Find Zero Point of Instrument](ZeroPointAnalysis/3-InstrumentZeroPoint/3-InstrumentZeroPoint.md) +- [Find the Best Exposure Meter Bin](ZeroPointAnalysis/4-BestExpMeterBin/4-BestExpMeterBin.md) +- [Examine the Distribution of Slowdown Factors](ZeroPointAnalysis/5-ETCs/5-ETCs.md) diff --git a/docs/status.md b/docs/status.md index d38fb1160..1f13a83eb 100644 --- a/docs/status.md +++ b/docs/status.md @@ -1,25 +1,53 @@ # Instrument Status -### Current and Past Announcements +### Current Announcements + +* 2026 April: [Turbo Pump Failure](announcements/2026-04-06_tubo_failure.md) +* 2026 February: [26A Status Announcement](announcements/2026-02-01_26A-status.md) + +KPF is OFFLINE until further notice due to the vacuum pump failure described in the 2026 April announcement above. A Failure Review Board is investigating the root cause of the pump failure and will advise the Observatory on a path forward. Until that report comes out, the schedule for KPF's return to service is uncertain. -* 2025 August: [26A Stability Announcement](KPF Stability Statement - August 15 2025.pdf) -* 2023 September: [Keck Science Meeting presentation](Keck Science Meeting 2023 Breakout Session.pdf) ### Status Summary by Subsystem -This is an attempt to summarize the status of various sub-systems of the instrument. Each sub-system name is color coded to indicate the status at a glance: green means functioning normally, orange means mostly normal, but with some caveats or minor issues, and red means the sub-system is compromised in some way. -Last Status Update: 2025-10-22 +This is an attempt to summarize the status of various sub-systems of the instrument. Each sub-system name is color coded to indicate the status at a glance: green means functioning normally, orange means mostly normal, but with some caveats or minor issues, and red means the sub-system is compromised in some way. -- **Detector Noise**: Starting in November of 2024, additional pattern noise has been present on the detectors. We have been working on eliminating the spurious nose, but we have been unable to completely remove it. As of late-October 2025 the read noise on the Green side remains elevated (~10 electrons), while the red side is near our nominal target (~4.3 electrons). -- **LFC**: We are evaluating the reliability of the LFC after recent service. Initial indications look prominsing on reliability, though the bluest flux (below ~490 nm) is not consistent. +- **Detector Noise**: Starting in November of 2024, additional non-gaussian noise has been present on the detectors. As of late-October 2025 the read noise on the Green side remains elevated (~10 electrons), while the red side is at our target level (~4.3 electrons). +- **LFC**: Initial evaluations of the reliability of the LFC after the recent service look promising, though the bluest flux (below ~490 nm) is not consistent. +- **Detector Cooling Systems**: The green side CCR has very little overhead on maintaining temperature and has quasi-periodic deviations which affect the detector. Red side is performing well, but has shown evidence of a slow degradation of performance. - **Etalon**: Operational. -- **Detector Cooling Systems**: Both detectors are now cooled with closed cycle refrigerators (CCRs). The green side CCR has problems and has very little overhead on maintaining temperature and has quasi-periodic deviations which affect the detector. Red side is performing well. -- **Detector Errors**: The red and green detectors suffer from occasional “start state errors” in which the affected detector remains in the start phase and does not produce a useful exposure. The observing scripts detect this, abort the current exposure (with read out) and start a fresh exposure on both cameras. **No action is necessary on the part of the observer.** The occurrence rate is such that around one in every 180 exposures is affected by one of the two detectors experiencing this error. +- **Detector Errors**: The red and green detectors suffer from occasional “start state errors” in which the affected detector does not produce a useful exposure. The observing scripts detect this, abort the exposure (with read out) and start a fresh exposure on both cameras. **No action is necessary on the part of the observer.** The occurrence rate is such that around one in every 180 exposures is affected by one of the two detectors experiencing this error. - **Ca H&K Detector**: The CA H&K detector is operational. - **Exposure Meter Terminated Exposures**: Operational. - **Tip Tilt Corrections**: The tip tilt axis are currently correcting as expected. - **Double Star Observations**: Operational. - **Simultaneous Calibration (SimulCal)**: Simultaneous calibrations are supported. -- **Nod to Sky Observations**: For observations which need a sky measurement other than the built in sky fibers, nodding away to a sky position can be accomplished manually by running separate OBs for the target and sky and asking the OA to offset the telescope as appropriate. We plan to build a separate Nod To Sky observing mode which will accomplish this within a single OB, but that is not yet available. - **Off Target Guiding**: Not yet commissioned. Currently, the tip tilt system must be able to detect the science target in order to position it on the fiber. + +Last Updated: 2026-02-01 + +### KPF Era 4.0 Temperature Stability Summary + +KPF Era 4.0 began in late-October 2025 after Servicing Mission 4. We continue to have issues with the cooling systems for the detectors, especially the Green side. We list below all temperature excursions in which one of the detectors deviated temperature by more than 5 mK from the set point. Excursions of order 1 K (1000 mK) or more are expected to induce a radial velocity offset which is not calibratable. We are providing this data as a guide, but users should not assume that past performance is a good indicator of future performance -- we have seen indications that the cooling systems are slowly degrading, so an increasing rate of these temperature excursions is a distinct possibility. + +| Side | Duration | Delta T | Start (HST) | End (HST) | +| ----- | ------------ | --------- | ------------------- | ------------------- | +| Green | 0.05 hours | 7 mK | 2025-11-09 21:22:14 | 2025-11-09 21:25:06 | +| Green | 0.36 hours | 109 mK | 2025-11-10 05:37:12 | 2025-11-10 05:58:50 | +| Green | 2.78 hours | 2,825 mK | 2025-11-10 11:09:06 | 2025-11-10 13:55:54 | +| Green | 0.63 hours | 250 mK | 2025-11-15 11:28:36 | 2025-11-15 12:06:32 | +| Green | 2.54 hours | 5,779 mK | 2025-11-16 14:27:20 | 2025-11-16 16:59:54 | +| Green | 4.04 hours | 6,147 mK | 2025-11-17 07:52:06 | 2025-11-17 11:54:40 | +| Green | 2.31 hours | 4,200 mK | 2025-11-18 03:40:28 | 2025-11-18 05:59:16 | +| Green | 3.14 hours | 5,862 mK | 2025-11-18 23:08:20 | 2025-11-19 02:16:52 | +| Green | 0.56 hours | 95 mK | 2026-01-23 21:12:46 | 2026-01-23 21:46:16 | +| Green | 1.20 hours | 712 mK | 2026-01-25 15:25:34 | 2026-01-25 16:37:48 | +| Green | 0.46 hours | 129 mK | 2026-01-30 19:36:24 | 2026-01-30 20:03:54 | +| Green | 13.82 hours | 53,527 mK | 2026-02-17 21:17:12 | 2026-02-18 11:06:40 | +| Green | 4.48 hours | 11,585 mK | 2026-02-19 02:40:56 | 2026-02-19 07:09:32 | + +### Past Announcements + +* 2025 August: [26A Stability Announcement](KPF Stability Statement - August 15 2025.pdf) +* 2023 September: [Keck Science Meeting presentation](Keck Science Meeting 2023 Breakout Session.pdf) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c5db4d66f..6fcea6c2c 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -7,6 +7,8 @@ * Scripts * [Existing Script is Running](#existing-script-is-running) * [Agitator Use is Disabled](#agitator-use-is-disabled) + * [Start of Night Script Failed](#start-of-night-script-failed) + * [End of Night Script Failed](#end-of-night-script-failed) * Calibrations * [Calibration Source is Not Working](#calibration-source-is-not-working) * [SlewCal or Simultaneous Calibration Source is Wrong](#slewcal-or-simultaneous-calibration-source-is-wrong) @@ -32,6 +34,7 @@ * SoCal * [Enclosure Lid Does not Move 1](#enclosure-lid-does-not-move-1) * [Enclosure Lid Does not Move 2](#enclosure-lid-does-not-move-2) + * [EKO Sun Tracker is not on Sun](#eko-sun-tracker-is-not-on-sun) # General Principles @@ -109,6 +112,75 @@ This means that the `kpfconfig.USEAGITATOR` keyword is set to “No”. This ke The agitator can be reenabled by simply setting the keyword to “Yes”. **This should only be done by WMKO staff** and should only be done if the agitator is fully functional. A broken or misbehaving agitator mechanism presents a significant danger to the science fibers. +## Start of Night Script Failed + +Symptom: + +When executing the start of night script, the script failed to read and set AO keywords. + +Problem: + +There is some kind of AO gateway communicaiton problem, and so far we don't know what the root cause is. This happens intermittently. + +Solution: + +Modify AO keywords as k1obsao in a k1aoserver-new terminal. + +Open AO hatch and check status: + +``` +modify -s ao aohatchcmd=open +show -s ao aohatchsts +``` + +Send PCU to KPF: + +``` +modify -s ao pcuname=KPF +show -s ao pcuname +``` + +Send AO rotator to 0 deg: + +``` +modify -s ao obrt=0 +show -s ao obrt +``` + +Set rotator to stationary: + +``` +modify -s dcs rotmode=stationary +show -s dcs rotmode +``` + +## End of Night Script Failed + +Symptom: + +When executing the end of night script, the script failed to read and set AO keywords. + +Problem: + +There is some kind of AO gateway communicaiton problem, and so far we don't know what the root causee is. This happens intermittently. + +Solution: + +Modify AO keywords as k1obsao in a k1aoserver-new terminal. + +Close AO hatch and check status: + +``` +modify -s ao aohatchcmd=close +show -s ao aohatchsts +``` + +Send AO rotator to 45 deg: + +``` +modify -s ao obrt=45 +show -s ao obrt +``` # Calibrations @@ -557,3 +629,20 @@ The `~/grep_for_dome_error` script will exclude many of the noisy, not useful li Solution: Reboot the controller (raspberry pi): `sudo reboot` and restart the kpfsocal3 dispatcher: `kpf restart kpfsocal3`. Multiple reboots may be required. + +## EKO Sun Tracker is not on Sun + +Symptom: + +The EKO solar tracker is not pointed at the sun, it may be parked. + +Problem: + +The tracker is not in the right mode. + +Solution: + +* Run `modify -s kpfsocal EKOCMD=2` which tells EKO to "guide" on the Sun. +* Run `modify -s kpfsocal EKOMODE=3` which sets the EKO in to "Sun-sensor with Learning" mode. + +If that fails, restart the EKO dispatcher: `kpf restart kpfsocal1` and run the commands again. diff --git a/kpf/OB_GUI/KPF_OB_GUI.py b/kpf/OB_GUI/KPF_OB_GUI.py index 6a47c3260..e2905c82b 100755 --- a/kpf/OB_GUI/KPF_OB_GUI.py +++ b/kpf/OB_GUI/KPF_OB_GUI.py @@ -82,7 +82,7 @@ def create_GUI_log(verbose=False): logdir.mkdir(mode=0o777, parents=True) LogFileName = logdir / 'OB_GUI_v2.log' LogFileHandler = RotatingFileHandler(LogFileName, - maxBytes=100*1024*1024, # 100 MB + maxBytes=20*1024*1024, # 20 MB backupCount=1000) # Keep old files LogFileHandler.setLevel(logging.DEBUG) LogFileHandler.setFormatter(LogFormat) @@ -131,8 +131,10 @@ def __init__(self, clargs, *args, **kwargs): # Determine git branch try: - cmd = 'git branch --show-current' + cmd = f'cd {Path(__file__).parent} ; git branch --show-current' result = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE) + self.log.debug('git branch --show-current') + self.log.debug(result.stdout.decode()) self.branch = result.stdout.decode().strip().strip('\n') self.log.debug(f'Got git branch result: {self.branch}') except: @@ -200,6 +202,7 @@ def __init__(self, clargs, *args, **kwargs): self.fast = False # Tracked values self.disabled_detectors = [] + self.disable_telescope_release_check = False self.telescope_released = GetTelescopeRelease.execute({}) # Get KPF Programs on schedule classical, cadence = GetScheduledPrograms.execute({'semester': 'current'}) @@ -303,9 +306,10 @@ def setupUi(self): self.SendOBListToMagiq = self.findChild(QtWidgets.QAction, 'actionSend_Current_OBs_as_Star_List') self.SendOBListToMagiq.triggered.connect(self.OBListModel.update_star_list) self.SendOBListToMagiq.setEnabled(False) - self.DisableMagiq = self.findChild(QtWidgets.QAction, 'actionDisable_Magiq') self.DisableMagiq.triggered.connect(self.toggle_magiq_enabled) + self.OverrideRelease = self.findChild(QtWidgets.QAction, 'actionOverride_Telescope_Release_Check') + self.OverrideRelease.triggered.connect(self.toggle_telescope_release_check) #------------------------------------------------------------------- # Main Window @@ -739,7 +743,10 @@ def update_LST(self, value): self.log.debug('Updating: SOB info, telescope_released') self.update_counter = 0 self.update_SOB_display() # Updates alt, az - self.telescope_released = GetTelescopeRelease.execute({}) + if self.disable_telescope_release_check is True: + self.telescope_released = True + else: + self.telescope_released = GetTelescopeRelease.execute({}) # Update execution history if we're vaguely near observing times try: UTh = int(self.UTValue.text().split(':')[0]) @@ -749,7 +756,6 @@ def update_LST(self, value): self.log.debug('Updating: execution history') self.refresh_history() - ##------------------------------------------- ## Methods for Observing Menu Actions ##------------------------------------------- @@ -845,6 +851,18 @@ def toggle_magiq_enabled(self): self.DisableMagiq.setText(action_text) self.update_selected_instrument(self.SelectedInstrument.text()) + def toggle_telescope_release_check(self): + self.log.info('Toggling telescope release check') + self.disable_telescope_release_check = not self.disable_telescope_release_check + self.log.debug(f"disable release check = {self.disable_telescope_release_check}") + action = {False: 'Disable', True: 'Enable'}[self.disable_telescope_release_check] + action_text = f"{action} Telescope Release Check" + self.OverrideRelease.setText(action_text) + if self.disable_telescope_release_check is True: + self.telescope_released = True + else: + self.telescope_released = GetTelescopeRelease.execute({}) + ##------------------------------------------- ## Methods to Operate on OB List UI @@ -1044,7 +1062,7 @@ def load_OBs_from_schedule(self, nonCCnight=False): else: semester, start, end = get_semester_dates(datetime.datetime.now()) utnow = datetime.datetime.utcnow() - date = utnow-datetime.timedelta(hours=20) # Switch dates at 10am HST, 2000UT + date = utnow-datetime.timedelta(hours=17) # Switch dates at 7am HST, 1700UT date_str = date.strftime('%Y-%m-%d').lower() if nonCCnight: schedule_files = [self.schedule_path / semester / date_str / f'full-{WB}' / 'output' / 'night_plan.csv' @@ -1124,6 +1142,11 @@ def load_OBs_from_schedule(self, nonCCnight=False): retrievedOBcount += 1 else: errs += failure_messages + try: + if type(errs[-1]) == list: + errs[-1] = errs[-1] + [entry['Target']] + except: + self.log.warning(f'Unable to append Target info to error: {entry["Target"]}') self.ProgressBar.setValue(int(scheduledOBcount/Nsched*100)) # Append a slewcal OB for convienience if self.OBcache['slewcal'] is not None: @@ -1136,7 +1159,17 @@ def load_OBs_from_schedule(self, nonCCnight=False): self.set_SortOrWeather() # Pop up for any errors if len(errs) > 0: - ConfirmationPopup('Errors retrieving OBs:', errs, info_only=True, warning=True).exec_() + if type(errs) == list: + msg = '' + for err in errs: + if type(err) == list: + msg += " ".join(err) + "\n" + else: + msg += f"{str(err)}\n" + else: + msg = str(errs) + self.log.warning('Errors when retrieving OBs:\n'+msg) + ConfirmationPopup('Errors retrieving OBs:', msg, info_only=True, warning=True).exec_() def refresh_history(self): self.log.debug(f"refresh_history") @@ -1231,8 +1264,12 @@ def set_SOB_Target(self, SOB): self.log.error(e) RA_str = SOB.Target.get('RA') Dec_str = SOB.Target.get('Dec') - self.SOB_TargetRALabel.setText('RA (Epoch=?):') - self.SOB_TargetDecLabel.setText('Dec (Epoch=?):') + if SOB.Target.get('Epoch') is not None: + RAlabel = f"RA (epoch={SOB.Target.Epoch}):" + DecLabel = f"Dec (epoch={SOB.Target.Epoch}):" + else: + RAlabel = 'RA (epoch=?):' + DecLabel = 'Dec (epoch=?):' # If proper motion values are set, try to propagate proper motions # if abs(SOB.Target.PMRA.value) > 0.001 or abs(SOB.Target.PMDEC.value) > 0.001: # try: @@ -1401,7 +1438,7 @@ def prepare_execution_history_file(self): self.execution_history_file = logdir / f'KPFCC_executions_{semester}.csv' if self.execution_history_file.exists() is False: with open(self.execution_history_file, 'w') as f: - contents = ['# timestamp', 'decimalUT', 'executedID', 'OB summary', + contents = ['timestamp', 'decimalUT', 'executedID', 'OB summary', 'executed_line', 'scheduleUT', 'schedule_current_line', 'scheduleUT_current', 'schedule_next_line', 'scheduleUT_next', diff --git a/kpf/OB_GUI/KPF_OB_GUI.ui b/kpf/OB_GUI/KPF_OB_GUI.ui index c315729a0..7f8914061 100644 --- a/kpf/OB_GUI/KPF_OB_GUI.ui +++ b/kpf/OB_GUI/KPF_OB_GUI.ui @@ -1745,12 +1745,13 @@ OB List + + - - + @@ -1781,6 +1782,7 @@ + @@ -1898,6 +1900,11 @@ Load KPF-CC Schedule for a non-CC Night + + + Disable Telescope Release Check + + diff --git a/kpf/OB_GUI/Popups.py b/kpf/OB_GUI/Popups.py index 1735d0bd9..036227067 100644 --- a/kpf/OB_GUI/Popups.py +++ b/kpf/OB_GUI/Popups.py @@ -23,7 +23,7 @@ def __init__(self, window_title, msg, info_only=False, warning=False, QtWidgets.QMessageBox.__init__(self, *args, **kwargs) self.setWindowTitle(window_title) if type(msg) == list: - msg = "\n".join(msg) + msg = ' '.join(msg) self.setText(msg) if info_only == True: self.setIcon(QtWidgets.QMessageBox.Information) diff --git a/kpf/ObservingBlocks/ObservationProperties.yaml b/kpf/ObservingBlocks/ObservationProperties.yaml index 44284d0a2..b04d9bb22 100644 --- a/kpf/ObservingBlocks/ObservationProperties.yaml +++ b/kpf/ObservingBlocks/ObservationProperties.yaml @@ -47,8 +47,8 @@ - name: 'ExpMeterThreshold' comment: '[Mphotons/angstrom] Flux at the science detector at peak of order at which to terminate the exposure' valuetype: float - defaultvalue: 50000 - precision: 0 + defaultvalue: 1 + precision: 2 - name: 'TakeSimulCal' comment: 'Inject simultaneous calibration light on to the detector during exposure?' valuetype: bool @@ -62,7 +62,7 @@ valuetype: str defaultvalue: 'OD 0.1' - name: 'CalND2' - comment: 'Cal Bench filter 2 position. Throughput = 10^-OD. Values: OD 0.1, OD 0.3, OD 0.5, OD 0.8, OD 1.0, OD 4.0' + comment: 'Cal Bench filter 2 position. Throughput = 10^-OD. Values: OD 0.1, OD 0.3, OD 0.5, OD 1.0, OD 1.3, OD 2.0' valuetype: str defaultvalue: 'OD 0.1' - name: 'NodN' diff --git a/kpf/ObservingBlocks/Target.py b/kpf/ObservingBlocks/Target.py index 47e780535..53ece4d00 100644 --- a/kpf/ObservingBlocks/Target.py +++ b/kpf/ObservingBlocks/Target.py @@ -110,7 +110,11 @@ def to_star_list(self): except: rastr = str(self.RA).replace(':', ' ') decstr = str(self.Dec).replace(':', ' ') - out = f"{self.TargetName.value:15s} {rastr} {decstr}" + if len(self.TargetName.value) > 15: + short_name = self.TargetName.value[:15] + else: + short_name = self.TargetName.value + out = f"{short_name:15s} {rastr} {decstr}" if str(self.Equinox) == 'J2000': out += f" 2000" else: diff --git a/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml b/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml index 758695150..c1664c037 100644 --- a/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml +++ b/kpf/ObservingBlocks/exampleOBs/Calibrations.yaml @@ -19,13 +19,13 @@ Calibrations: - CalSource: 'EtalonFiber' Object: 'Etalon' nExp: 1 - ExpTime: 20 + ExpTime: 60 TriggerCaHK: False TriggerGreen: True TriggerRed: True IntensityMonitor: False CalND1: 'OD 0.1' - CalND2: 'OD 1.3' + CalND2: 'OD 1.0' OpenScienceShutter: True OpenSkyShutter: True TakeSimulCal: True @@ -34,12 +34,12 @@ Calibrations: - CalSource: 'LFCFiber' Object: 'LFC' nExp: 1 - ExpTime: 60 + ExpTime: 10 TriggerCaHK: False TriggerGreen: True TriggerRed: True IntensityMonitor: False - CalND1: 'OD 1.0' + CalND1: 'OD 1.3' CalND2: 'OD 0.1' OpenScienceShutter: True OpenSkyShutter: True @@ -60,21 +60,6 @@ Calibrations: OpenSkyShutter: True TakeSimulCal: True WideFlatPos: 'Blank' -# Th Daily (for CaHK): -- CalSource: 'Th_daily' - Object: 'ThAr_forCaHK' - nExp: 1 - ExpTime: 120 - TriggerCaHK: False - TriggerGreen: True - TriggerRed: True - IntensityMonitor: False - CalND1: 'OD 0.1' - CalND2: 'OD 0.1' - OpenScienceShutter: True - OpenSkyShutter: True - TakeSimulCal: True - WideFlatPos: 'Blank' # U Daily: - CalSource: 'U_daily' Object: 'UNe' diff --git a/kpf/ao/SetupAOforKPF.py b/kpf/ao/SetupAOforKPF.py index e004e5142..e6f6e48dd 100644 --- a/kpf/ao/SetupAOforKPF.py +++ b/kpf/ao/SetupAOforKPF.py @@ -1,4 +1,7 @@ +import subprocess + import ktl +import ktl.Exceptions from kpf import log, cfg from kpf.exceptions import * @@ -43,27 +46,39 @@ def pre_condition(cls, args): @classmethod def perform(cls, args): - log.info('Set AO rotator to Manual') - SetAORotatorManual.execute({}) - - log.info('Set AO rotator to 0 deg') - SetAORotator.execute({'dest': 0}) - - log.info('Turn off HEPA') - TurnHepaOff.execute({}) - - log.info('Set AO in DCS sim mode') - SetAODCStoSIM.execute({}) - - log.info('Turn K1 AO light source off') - TurnLightSourceOff.execute({}) - PCSstagekw = ktl.cache('ao', 'PCSFNAME') - if PCSstagekw.read() != 'kpf': - log.info('Move PCU to Home') - SendPCUtoHome.execute({}) - log.info('Move PCU to KPF') - SendPCUtoKPF.execute({}) + try: + log.info('Set AO rotator to Manual') + SetAORotatorManual.execute({}) + + log.info('Set AO rotator to 0 deg') + SetAORotator.execute({'dest': 0}) + + log.info('Turn off HEPA') + TurnHepaOff.execute({}) + + log.info('Set AO in DCS sim mode') + SetAODCStoSIM.execute({}) + + log.info('Turn K1 AO light source off') + TurnLightSourceOff.execute({}) + + PCSstagekw = ktl.cache('ao', 'PCSFNAME') + if PCSstagekw.read() != 'kpf': + log.info('Move PCU to KPF') + SendPCUtoKPF.execute({}) + except Exception as e: + log.warning('SetupAOforKPF failed.') + log.warning(e) + log.warning(f'SSHing to k1obsao@k1aoserver-new to run kpfStart.csh') + ssh_cmds = ['ssh -X k1obsao@k1aoserver-new kpfStart.csh', + f'echo "Done!"', + f'sleep 30'] + ssh_cmd = ' ; '.join(ssh_cmds) + cmd = ['xterm', '-title', 'SetupAOforKPF', '-name', 'SetupAOforKPF', + '-fn', '10x20', '-bg', 'black', '-fg', 'white', + '-e', f'{ssh_cmd}'] + proc = subprocess.Popen(cmd) @classmethod def post_condition(cls, args): diff --git a/kpf/ao/ShutdownAOforKPF.py b/kpf/ao/ShutdownAOforKPF.py new file mode 100644 index 000000000..e00d8fed8 --- /dev/null +++ b/kpf/ao/ShutdownAOforKPF.py @@ -0,0 +1,63 @@ +import subprocess + +import ktl +import ktl.Exceptions + +from kpf import log, cfg +from kpf.exceptions import * +from kpf.KPFTranslatorFunction import KPFFunction, KPFScript +from kpf.ao.ControlAOHatch import ControlAOHatch +from kpf.ao.TurnHepaOn import TurnHepaOn + + +class ShutdownAOforKPF(KPFFunction): + ''' + + KTL Keywords Used: + + - `` + + Functions Called: + + - `` + ''' + @classmethod + def pre_condition(cls, args): + pass + + @classmethod + def perform(cls, args): + log.info('Closing AO Hatch') + try: + ControlAOHatch.execute({'destination': 'closed'}) + except Exception as e: + log.warning(f"Failure controlling AO hatch") + log.warning(e) + log.warning(f'SSHing to k1obsao@k1aoserver-new to run modify -s ao aohatchcmd=1') + ssh_cmds = ["ssh k1obsao@k1aoserver-new 'modify -s ao aohatchcmd=1'", + f'echo "Done!"', + f'sleep 30'] + ssh_cmd = ' ; '.join(ssh_cmds) + cmd = ['xterm', '-title', 'CloseAOHatch', '-name', 'CloseAOHatch', + '-fn', '10x20', '-bg', 'black', '-fg', 'white', + '-e', f'{ssh_cmd}'] + proc = subprocess.Popen(cmd) + log.info('Turning on AO HEPA Filter System') + try: + TurnHepaOn.execute({}) + except Exception as e: + log.warning(f"Failure controlling AO HEPA Filter System") + log.warning(e) + log.warning(f'SSHing to k1obsao@k1aoserver-new to run modify -s ao obhpaon=1') + ssh_cmds = ["ssh k1obsao@k1aoserver-new 'modify -s ao obhpaon=1'", + f'echo "Done!"', + f'sleep 30'] + ssh_cmd = ' ; '.join(ssh_cmds) + cmd = ['xterm', '-title', 'TurnHEPAOn', '-name', 'TurnHEPAOn', + '-fn', '10x20', '-bg', 'black', '-fg', 'white', + '-e', f'{ssh_cmd}'] + proc = subprocess.Popen(cmd) + + @classmethod + def post_condition(cls, args): + pass diff --git a/kpf/ao/TurnHepaOff.py b/kpf/ao/TurnHepaOff.py index de7f02ed2..e3dc38a1f 100644 --- a/kpf/ao/TurnHepaOff.py +++ b/kpf/ao/TurnHepaOff.py @@ -25,6 +25,7 @@ def perform(cls, args): @classmethod def post_condition(cls, args): - OBHPASTA = ktl.cache('ao', 'OBHPASTA') - if OBHPASTA.waitfor('== "off"', timeout=3) is not True: - raise FailedToReachDestination(OBHPASTA.read(), 'off') + pass +# OBHPASTA = ktl.cache('ao', 'OBHPASTA') +# if OBHPASTA.waitfor('== "off"', timeout=3) is not True: +# raise FailedToReachDestination(OBHPASTA.read(), 'off') diff --git a/kpf/ao/TurnHepaOn.py b/kpf/ao/TurnHepaOn.py index 735adf158..71d47978d 100644 --- a/kpf/ao/TurnHepaOn.py +++ b/kpf/ao/TurnHepaOn.py @@ -25,6 +25,7 @@ def perform(cls, args): @classmethod def post_condition(cls, args): - OBHPASTA = ktl.cache('ao', 'OBHPASTA') - if OBHPASTA.waitfor('== "on"', timeout=3) is not True: - raise FailedToReachDestination(OBHPASTA.read(), 'on') + pass +# OBHPASTA = ktl.cache('ao', 'OBHPASTA') +# if OBHPASTA.waitfor('== "on"', timeout=3) is not True: +# raise FailedToReachDestination(OBHPASTA.read(), 'on') diff --git a/kpf/calbench/ListLFCRuns.py b/kpf/calbench/ListLFCRuns.py index 726c1636b..718b257e1 100644 --- a/kpf/calbench/ListLFCRuns.py +++ b/kpf/calbench/ListLFCRuns.py @@ -36,14 +36,15 @@ class ListLFCRuns(KPFFunction): @classmethod def perform(cls, args): ndays = args.get('ndays', 1) - now = datetime.datetime.now() if args.get('date', '') in ['', 'now']: - start = now - datetime.timedelta(days=ndays) + start = datetime.datetime.now() - datetime.timedelta(days=ndays) + end = datetime.datetime.now() else: start = datetime.datetime.strptime(args.get('date'), '%Y-%m-%d') - datetime.timedelta(days=ndays) + end = datetime.datetime.strptime(args.get('date'), '%Y-%m-%d') LFC_history = keygrabber.retrieve({'kpfcal': ['OPERATIONMODE']}, begin=start.timestamp(), - end=now.timestamp()) + end=end.timestamp()) astrocomb = [x for x in LFC_history if x['ascvalue'] == 'AstroComb'] # kws = {'kpfcal': ['OPERATIONMODE', 'POS_INTENSITY', 'SPECFLATIR', diff --git a/kpf/cli_interface.py b/kpf/cli_interface.py index 169b39762..0caa61271 100644 --- a/kpf/cli_interface.py +++ b/kpf/cli_interface.py @@ -258,14 +258,17 @@ def main(table_loc, parsed_args, function_args, kpfdo_parser): help="The OB file to run.") parser.add_argument("-d", "--obid", "--id", dest="obid", type=str, help="The unique database ID of the OB to run.") + logger.debug('Added script -f and -d to parser') if parsed_args.help is True: + logger.debug('Printing help to screen') print(' '+function.__doc__) help_str = parser.format_help() help_str = help_str.replace('usage: kpfdo', f'usage: kpfdo {function_args[0]}') print(help_str) return + logger.debug('Parsing final set of arguments') try: # Append these parsed args onto whatever was (or wasn't) # found in the input file (i.e. if -f was used) @@ -276,8 +279,14 @@ def main(table_loc, parsed_args, function_args, kpfdo_parser): logger.error(e) logger.error(traceback.format_exc()) sys.exit(1) + except Exception as e: + logger.error('Failed to parse arguments') + logger.error(e) + logger.error(traceback.format_exc()) + sys.exit(1) if script is True: + logger.debug(f'Checking for OB input') OB = None input_file_string = parsed_func_args.get('file', None) if input_file_string is not None: diff --git a/kpf/engineering/TriggerGreenMiniFill.py b/kpf/engineering/TriggerGreenMiniFill.py deleted file mode 100644 index f7cc25030..000000000 --- a/kpf/engineering/TriggerGreenMiniFill.py +++ /dev/null @@ -1,57 +0,0 @@ -import time - -import ktl - -from kpf.KPFTranslatorFunction import KPFTranslatorFunction -from kpf import (log, KPFException, FailedPreCondition, FailedPostCondition, - FailedToReachDestination, check_input) -from kpf.utils.SendEmail import SendEmail - - -##------------------------------------------------------------------------- -## TriggerGreenMiniFill -##------------------------------------------------------------------------- -class TriggerGreenMiniFill(KPFTranslatorFunction): - '''I really hope this is not necessary in the long term. - ''' - @classmethod - def pre_condition(cls, args, logger, cfg): - kpffill = ktl.cache('kpffill') - if kpffill['GREENFILLIP'].read() == 'True': - raise FailedPreCondition('Green fill already in progress') - - @classmethod - def perform(cls, args, logger, cfg): - kpffill = ktl.cache('kpffill') - # Start fill - log.warning(f'Starting green mini fill') - kpffill['GREENSTART'].write(1) - # Wait - sleep_time = args.get('duration', 240) - log.debug(f'Sleeping {sleep_time:.0f} s') - time.sleep(sleep_time) - # Stop fill - if kpffill['GREENFILLIP'].read() == 'True': - log.warning(f'Stopping green mini fill') - kpffill['GREENSTOP'].write(1) - time.sleep(5) - else: - msg = 'Expected green mini fill to be in progress.' - SendEmail.execute({'Subject': 'TriggerGreenMiniFill Failed', - 'Message': f'{msg}'}) - raise KPFException(msg) - - @classmethod - def post_condition(cls, args, logger, cfg): - kpffill = ktl.cache('kpffill') - if kpffill['GREENFILLIP'].read() == 'True': - msg = 'Green still in progress, should be stopped!' - SendEmail.execute({'Subject': 'TriggerGreenMiniFill Failed', - 'Message': f'{msg}'}) - raise FailedPostCondition(msg) - - @classmethod - def add_cmdline_args(cls, parser, cfg=None): - parser.add_argument('duration', type=float, - help='The duration of the fill in seconds (240 recommended)') - return super().add_cmdline_args(parser, cfg) diff --git a/kpf/engineering/TriggerRedMiniFill.py b/kpf/engineering/TriggerRedMiniFill.py deleted file mode 100644 index 1d09a20e1..000000000 --- a/kpf/engineering/TriggerRedMiniFill.py +++ /dev/null @@ -1,57 +0,0 @@ -import time - -import ktl - -from kpf.KPFTranslatorFunction import KPFTranslatorFunction -from kpf import (log, KPFException, FailedPreCondition, FailedPostCondition, - FailedToReachDestination, check_input) -from kpf.utils.SendEmail import SendEmail - - -##------------------------------------------------------------------------- -## TriggerRedMiniFill -##------------------------------------------------------------------------- -class TriggerRedMiniFill(KPFTranslatorFunction): - '''I really hope this is not necessary in the long term. - ''' - @classmethod - def pre_condition(cls, args, logger, cfg): - kpffill = ktl.cache('kpffill') - if kpffill['REDFILLIP'].read() == 'True': - raise FailedPreCondition('Red fill already in progress') - - @classmethod - def perform(cls, args, logger, cfg): - kpffill = ktl.cache('kpffill') - # Start fill - log.warning(f'Starting Red mini fill') - kpffill['REDSTART'].write(1) - # Wait - sleep_time = args.get('duration', 240) - log.debug(f'Sleeping {sleep_time:.0f} s') - time.sleep(sleep_time) - # Stop fill - if kpffill['REDFILLIP'].read() == 'True': - log.warning(f'Stopping Red mini fill') - kpffill['REDSTOP'].write(1) - time.sleep(5) - else: - msg = 'Expected Red mini fill to be in progress.' - SendEmail.execute({'Subject': 'TriggerRedMiniFill Failed', - 'Message': f'{msg}'}) - raise KPFException(msg) - - @classmethod - def post_condition(cls, args, logger, cfg): - kpffill = ktl.cache('kpffill') - if kpffill['RedFILLIP'].read() == 'True': - msg = 'Red still in progress, should be stopped!' - SendEmail.execute({'Subject': 'TriggerRedMiniFill Failed', - 'Message': f'{msg}'}) - raise FailedPostCondition(msg) - - @classmethod - def add_cmdline_args(cls, parser, cfg=None): - parser.add_argument('duration', type=float, - help='The duration of the fill in seconds (240 recommended)') - return super().add_cmdline_args(parser, cfg) diff --git a/kpf/engineering/analysis/AnalyzeKPFCCExecutions.py b/kpf/engineering/analysis/AnalyzeKPFCCExecutions.py new file mode 100644 index 000000000..0217980aa --- /dev/null +++ b/kpf/engineering/analysis/AnalyzeKPFCCExecutions.py @@ -0,0 +1,169 @@ +import sys +from pathlib import Path +from datetime import datetime, timedelta +import re + +from astropy.table import Table, Column +import numpy as np + +from matplotlib import pyplot as plt +from matplotlib import ticker + + +semester = '2026A' +logdir = Path(f'/s/sdata1701/KPFTranslator_logs/') +execution_history_file = logdir / f'KPFCC_executions_{semester}.csv' +executions = Table.read(execution_history_file, format='ascii.csv') +print(f'Loaded {len(executions)} executions in {semester}') + +remove_rows = [] +timestamps = [] +line_deltas = [] +time_deltas = [] +UTnight_strings = [] +CNdelta = [] +for r,ex in enumerate(executions): + + # Remove Calibration or filler rows + if re.search('Calibration', ex['OB summary']) is not None: + remove_rows.append(r) +# elif int(ex['scheduleUT']) in [0, 24]: +# print(f"Removing {ex['OB summary']} because scheduled time is {ex['scheduleUT']}") +# remove_rows.append(r) + else: + try: + # Add timestamps + timestamps.append(datetime.strptime(ex['timestamp'], '%Y-%m-%d %H:%M:%S UT')) + # Add line delta + line_deltas.append(ex['executed_line'] - ex['schedule_current_line'] - 0.5) + # Add time deltas + time_deltas.append(ex['decimalUT']-ex['scheduleUT']) + # Add night string + UTnight_strings.append(ex['timestamp'].split()[0]) + # Check current, next lines are 1 apart + CNdelta.append(ex['schedule_next_line'] - ex['schedule_current_line']) + except Exception as e: + print('Failed to add supplementary metadata') + print(e) + print() + print(ex.keys()) + print(ex) + raise e + +print(f"Removing {len(remove_rows)} entries") +executions.remove_rows(remove_rows) +executions.add_column(Column(timestamps, name='datetime')) +executions.add_column(Column(line_deltas, name='line_delta')) +executions.add_column(Column(time_deltas, name='time_delta')) +executions.add_column(Column(UTnight_strings, name='UTnight')) +executions.add_column(Column(CNdelta, name='CNdelta')) + +# print(executions.keys()) +# print(executions['executed_line', 'schedule_current_line', 'schedule_next_line', 'on_schedule'][:9]) +# print(executions['decimalUT', 'scheduleUT', 'scheduleUT_current', 'scheduleUT_next', 'on_schedule'][:9]) +# print(executions['datetime', 'line_delta', 'time_delta', 'UTnight', 'on_schedule'][:9]) + +on_schedule_groups = executions.group_by('on_schedule') +off_schedule = on_schedule_groups.groups[0] +off_schedule = off_schedule[off_schedule['scheduleUT'] < 23.99] +on_schedule = on_schedule_groups.groups[1] +unscheduled = executions[executions['scheduleUT'] > 23.99] +PctOffSched = len(off_schedule)/len(executions) +PctOnSched = len(on_schedule)/len(executions) +PctUnSched = len(unscheduled)/len(executions) +print(f"In {semester}, {len(executions)} KPF-CC science OBs have been successfully executed.") +print(f" {len(on_schedule):4d} scheduled OBs were executed according to the schedule ({PctOnSched:.1%})") +print(f" {len(off_schedule):4d} scheduled OBs were executed off the prescribed schedule ({PctOffSched:.1%})") +print(f" {len(unscheduled):4d} unscheduled OBs were executed ({PctUnSched:.1%})") + + +oddities = executions[(executions['CNdelta'] != 1) & (executions['schedule_next_line'] > -1)] +# print(oddities['UTnight', 'decimalUT', 'schedule_current_line', 'schedule_next_line']) +# print(np.median(oddities['CNdelta']), np.min(oddities['CNdelta']), np.max(oddities['CNdelta'])) + +# Time Delta Plot +plt.figure(figsize=(10,6)) +plt.title('Distribution of Time Offsets from Schedule') +plt.subplot(2,1,1) +timebins_on = np.arange(-3,+3,0.10) +plt.hist(on_schedule['time_delta'], bins=timebins_on, + color='g', alpha=0.4, label='On Schedule') +plt.axvline(0, color='k') +plt.legend(loc='best') +plt.ylabel('N Executions') + +plt.subplot(2,1,2) +timebins_off = np.arange(-3,+3,0.10) +plt.hist(off_schedule['time_delta'], bins=timebins_off, + color='r', alpha=0.4, label='Off Schedule') +plt.axvline(0, color='k') +plt.legend(loc='best') +plt.ylabel('N Executions') + +plt.xlabel('Time Delta (hours) [actual-scheduled]') +plt.savefig(f'KPF-CC_TimeOffsetDistribution_{semester}.png', bbox_inches='tight', pad_inches=0.1) + + +# Line Delta Plot +# plt.figure(figsize=(10,6)) +# plt.title('Distribution of Schedule Line Offsets') +# plt.subplot(2,1,1) +# timebins_on = np.arange(-15.5,+14.5,1) +# plt.hist(on_schedule['line_delta'], bins=timebins_on, color='g', alpha=0.4) +# plt.axvline(0, color='k') +# plt.ylabel('N Executions') +# +# plt.subplot(2,1,2) +# timebins_off = np.arange(-15.5,+14.5,1) +# plt.hist(off_schedule['line_delta'], bins=timebins_off, color='r', alpha=0.4) +# plt.axvline(0, color='k') +# plt.ylabel('N Executions') +# +# plt.xlabel('Schedule Delta (lines) [actual-scheduled]') +# plt.savefig('KPF-CC_LineOffsetDistribution.png', bbox_inches='tight', pad_inches=0.1) +# plt.show() + + +# On Schedule Rate Over Semester +frac_on = [] +frac_off = [] +frac_un = [] +Ntot = [] +UTnights = [] +UTnight_groups = executions.group_by('UTnight') +for key, UTnight_group in zip(UTnight_groups.groups.keys, UTnight_groups.groups): + UTnights.append(key['UTnight']) + Ntot.append(len(UTnight_group)) + + N_on_schedule = len(UTnight_group[UTnight_group['on_schedule'] == 'True']) + N_off_schedule = len(UTnight_group[UTnight_group['on_schedule'] == 'False']) + N_unscheduled = len(UTnight_group[UTnight_group['scheduleUT'] > 23.99]) + N_off_schedule -= N_unscheduled + + frac_off.append(N_off_schedule/len(UTnight_group)) + frac_on.append(N_on_schedule/len(UTnight_group)) + frac_un.append(N_unscheduled/len(UTnight_group)) + +plt.figure(figsize=(10,6)) +plt.title('Fraction of On Schedule (green), Off Schedule (red), and Unscheduled (Magenta) OBs') +plt.bar(range(1,len(UTnights)+1,1), frac_on, color='g', alpha=0.6) +plt.bar(range(1,len(UTnights)+1,1), frac_off, color='r', alpha=0.6, + bottom=frac_on) +plt.bar(range(1,len(UTnights)+1,1), frac_un, color='m', alpha=0.6, + bottom=np.array(frac_on)+np.array(frac_off)) + +plt.axhline(PctOnSched, color='g', label=f'On {PctOnSched:.1%}') +plt.axhline(PctOffSched, color='r', label=f'Off {PctOffSched:.1%}') +plt.axhline(PctUnSched, color='m', label=f'Un {PctUnSched:.1%}') + +for i,N in enumerate(Ntot): + plt.text(i+0.60, 1.03, f"N={N}", rotation=90) + +plt.ylim(0,1.15) +plt.xlim(0,len(UTnights)+10) +plt.legend(loc='best') +plt.gca().xaxis.set_major_locator(ticker.MultipleLocator(1.0)) +tick_labels = ['', ''] + [UTN[5:] for UTN in UTnights] +plt.gca().set_xticklabels(tick_labels, rotation=90) +plt.savefig(f'KPF-CC_OnScheduleRate_{semester}.png', bbox_inches='tight', pad_inches=0.1) +# plt.show() \ No newline at end of file diff --git a/kpf/engineering/analysis/ListTemperatureExcursions.py b/kpf/engineering/analysis/ListTemperatureExcursions.py new file mode 100644 index 000000000..3ed61c281 --- /dev/null +++ b/kpf/engineering/analysis/ListTemperatureExcursions.py @@ -0,0 +1,164 @@ +#!python3 + +## Import General Tools +import sys +import copy +import time +from datetime import datetime, timedelta +from matplotlib import pyplot as plt +import numpy as np +from scipy.signal import find_peaks + +import keygrabber + + +def find_excursions(side, threshold=0.005, min_duration=1*60*60, set_point=-100, + start='2025-11-01'): + excursion_min_duration = timedelta(hours=1) + set_point = -100 + + start_date = datetime.strptime(start, '%Y-%m-%d') + begin = time.mktime(start_date.timetuple()) + end = time.mktime(datetime.now().timetuple()) + +# start_date = datetime.strptime('2025-11-09 05:00:00', '%Y-%m-%d %H:%M:%S') +# begin = time.mktime(start_date.timetuple()) +# end_date = datetime.strptime('2025-11-10 16:00:00', '%Y-%m-%d %H:%M:%S') +# end = time.mktime(end_date.timetuple()) + + det_temp_history = keygrabber.retrieve({f'kpf{side.lower()}': ['STA_CCDVAL']}, + begin=begin, end=end) + det_temp = np.array([abs(float(x['binvalue'])-set_point) + for x in det_temp_history]) + det_time = np.array([datetime.fromtimestamp(x['time']) + for x in det_temp_history]) + + # Find Our of Range (ooR) time spans + events = [{'Begin': None, 'End': None, 'Side': side}] + ooR = det_temp > threshold + for i,val in enumerate(ooR): + if val == True and events[-1]['Begin'] is None: + events[-1]['Begin'] = det_time[i] + if val == False and events[-1]['End'] is None and events[-1]['Begin'] is not None: + events[-1]['End'] = det_time[i] + events.append({'Begin': None, 'End': None, 'Side': side}) + + if events[-1] == {'Begin': None, 'End': None, 'Side': side}: + events.pop(-1) + + merge = [] + for i,event in enumerate(events): + if i > 0: + ok_duration = (event['Begin']-events[i-1]['End']).total_seconds() + if ok_duration < min_duration: + merge.append(i) + merge.reverse() + for m in merge: + events[m-1]['End'] = events[m]['End'] + events = [e for i,e in enumerate(events) if i not in merge] + + # Find Peak on a Per event basis + for i,event in enumerate(events): + when = (det_time >= event['Begin']) & (det_time <= event['End']) + events[i]['DeltaTemp'] = max(det_temp[when]) + + return det_time, det_temp, events + +# # Find local maxima (peaks) +# peaks_indices, _ = find_peaks(det_temp, height=threshold, +# distance=1, width=5) +# +# # Get the actual values of the peaks +# peak_values = det_temp[peaks_indices] +# peak_times = det_time[peaks_indices] +# +# +# # Print the indices and values of the peaks +# remove_inds = [] +# for i,peak in enumerate(peak_values): +# pt = peak_times[i] +# print(f"{peak_times[i]}: {peak:.3f} K") +# delta_ts = abs(pt-peak_times) +# in_window = delta_ts < excursion_min_duration +# in_window_inds = np.where(in_window)[0] +# windowed_peak_values = copy.deepcopy(peak_values) +# windowed_peak_values[~in_window] = 0 +# max_ind = np.argmax(windowed_peak_values) +# not_peak = (in_window_inds != max_ind) +# remove_inds.extend(in_window_inds[not_peak]) +# +# final_times = [] +# final_values = [] +# for i,peak in enumerate(peak_values): +# if i not in remove_inds: +# final_times.append(peak_times[i]) +# final_values.append(peak_values[i]) +# +# return det_time, det_temp, final_times, final_values, events + + +if __name__ == '__main__': +# Gdet_time, Gdet_temp, Gfinal_times, Gfinal_values, Gevents = find_excursions('Green') +# Rdet_time, Rdet_temp, Rfinal_times, Rfinal_values, Revents = find_excursions('Red') + Gdet_time, Gdet_temp, Gevents = find_excursions('Green') + Rdet_time, Rdet_temp, Revents = find_excursions('Red') + +# Excursions = [] +# GreenExcursions = list(zip(Gfinal_times, Gfinal_values, ['G']*len(Gfinal_values))) +# if GreenExcursions: Excursions.extend(GreenExcursions) +# RedExcursions = list(zip(Rfinal_times, Rfinal_values, ['R']*len(Gfinal_values))) +# if RedExcursions: Excursions.extend(RedExcursions) +# Excursions = sorted(Excursions) +# +# print() +# print(f'List of Temperature Excursion Events') +# print('| Side | Date & Time (HST) | Delta T |') +# print('| ---- | ------------------- | -------- |') +# for i,entry in enumerate(Excursions): +# time_str = entry[0].strftime('%Y-%m-%d %H:%M:%S') +# print(f"| {entry[2]:4s} | {time_str} | {entry[1]*1000:5,.0f} mK |") + + events = Gevents + Revents + events = sorted(events, key=lambda x: x['Begin']) + + print(f'List of Temperature Excursion Events') + print('| Side | Duration | Delta T | Start (HST) | End (HST) |') + print('| ----- | ------------ | --------- | ------------------- | ------------------- |') + for event in events: + b = event['Begin'].strftime('%Y-%m-%d %H:%M:%S') + e = event['End'].strftime('%Y-%m-%d %H:%M:%S') + d = (event['End']-event['Begin']).total_seconds() / 3600 + dT = event['DeltaTemp']*1000 + print(f"| {event['Side']:4s} | {d:6.2f} hours | {dT:6,.0f} mK | {b:19s} | {e:19s} |") + + + # Plot the results to visualize + plt.figure(figsize=(12, 5)) + + plt.subplot(2,1,1) + plt.title('Detector Temperature Excursions') + plt.fill_between([min(Gdet_time), max(Gdet_time)], y1=0.0001, y2=0.005, color='g', alpha=0.3) + plt.plot(Gdet_time, Gdet_temp, 'k-', alpha=0.3, label='Green Temperature') + for Gevent in Gevents: + plt.fill_between([Gevent['Begin'], Gevent['End']], y1=0.005, y2=Gevent['DeltaTemp'], color='r', alpha=0.3) +# plt.scatter(Gfinal_times, Gfinal_values, color='red', marker='x', label='Green Excursions') + plt.ylim(0.0005, max([0.010, max(Gdet_temp)*2])) + plt.xlim(min(Gdet_time), max(Gdet_time)) + plt.legend() + plt.gca().set_yscale('log') + plt.ylabel('Temperature Delta (K)') + + plt.subplot(2,1,2) + plt.fill_between([min(Rdet_time), max(Rdet_time)], y1=0.0001, y2=0.005, color='g', alpha=0.3) + plt.plot(Rdet_time, Rdet_temp, 'k-', alpha=0.3, label='Red Temperature') + for Revent in Revents: + plt.fill_between([Revent['Begin'], Revent['End']], y1=0.005, y2=0.050, color='r', alpha=0.3) +# plt.scatter(Rfinal_times, Rfinal_values, color='red', marker='x', label='Red Excursions') + plt.xlim(min(Rdet_time), max(Rdet_time)) + plt.ylim(0.0005, max([0.010, max(Rdet_temp)*2])) + plt.legend() + plt.gca().set_yscale('log') + + plt.xlabel('Time') + plt.ylabel('Temperature Delta (K)') + plt.show() diff --git a/kpf/engineering/analysis/PlotBasementTemperatures.py b/kpf/engineering/analysis/PlotBasementTemperatures.py new file mode 100644 index 000000000..2b70c4cd3 --- /dev/null +++ b/kpf/engineering/analysis/PlotBasementTemperatures.py @@ -0,0 +1,90 @@ +#!python3 + +## Import General Tools +import sys +import copy +import time +from datetime import datetime, timedelta +from matplotlib import pyplot as plt +import numpy as np +from scipy.signal import find_peaks +from astropy.table import Table + +import keygrabber + + +def retrieve_basement_temperatures(start='2026-01-01', end='2026-02-01'): + + start_date = datetime.strptime(start, '%Y-%m-%d') + begin = time.mktime(start_date.timetuple()) + end_date = datetime.strptime(end, '%Y-%m-%d') + end = time.mktime(end_date.timetuple()) + +# start_date = datetime.strptime('2025-11-09 05:00:00', '%Y-%m-%d %H:%M:%S') +# begin = time.mktime(start_date.timetuple()) +# end_date = datetime.strptime('2025-11-10 08:00:00', '%Y-%m-%d %H:%M:%S') +# end = time.mktime(end_date.timetuple()) + + keywords = {f'kpfmet': ['TEMP', 'ETALON_AMBIENT', 'CAL_BENCH_BOT', + 'ENCLOSURE_FRONT', 'ENCLOSURE_TOP']} + temp_history = keygrabber.retrieve(keywords, begin=begin, end=end) + + data = {} + for kw in keywords['kpfmet']: + data[f'{kw} value'] = [] + data[f'{kw} times'] = [] + for entry in temp_history: + kw = entry['keyword'] + value = float(entry['binvalue']) + timestamp = datetime.fromtimestamp(entry['time']) + if value > -10 and value < 50: + data[f'{kw} value'].append(value) + data[f'{kw} times'].append(timestamp) + + return data, keywords['kpfmet'] + + +def plot_basement_temperatures(data, kws, start=None, end=None): + if start and end: + start_date = datetime.strptime(start, '%Y-%m-%d') + end_date = datetime.strptime(end, '%Y-%m-%d') + + # Plot the results to visualize + plt.figure(figsize=(14, 8)) + + plt.subplot(2,1,1) + plt.title('Basement Temperatures') + for kw in kws: + plt.plot(data[f'{kw} times'], data[f'{kw} value'], marker=',', label=kw) + plt.ylabel('Temperature (C)') + plt.grid() + plt.legend(loc='best') + plt.ylim(17.5,27.5) + if start and end: + plt.xlim(start_date, end_date) + + plt.subplot(2,1,2) + minmax = [0, 0] + for kw in kws: + mint = min(data[f'{kw} value']-np.median(data[f'{kw} value'])) + maxt = max(data[f'{kw} value']-np.median(data[f'{kw} value'])) + plt.plot(data[f'{kw} times'], data[f'{kw} value']-np.median(data[f'{kw} value']), + marker=',', label=kw, alpha=0.3) + if mint < minmax[0]: minmax[0] = np.floor(mint*4)/4 + if maxt > minmax[1]: minmax[1] = np.ceil(maxt*4)/4 + plt.ylabel('Temperature Changes (C)') + plt.grid() + plt.legend(loc='best') + plt.ylim(*minmax) + if start and end: + plt.xlim(start_date, end_date) + + plt.xlabel('Time') + plt.show() + + +if __name__ == '__main__': + start = '2025-11-01' + end = '2026-03-01' + data, kws = retrieve_basement_temperatures(start=start, end=end) + plot_basement_temperatures(data, kws, start=start, end=end) diff --git a/kpf/engineering/plot_ffts.py b/kpf/engineering/plot_ffts.py index 7935248a3..a62b106f1 100755 --- a/kpf/engineering/plot_ffts.py +++ b/kpf/engineering/plot_ffts.py @@ -1,6 +1,7 @@ #!python3 ## Import General Tools +import sys from pathlib import Path import argparse import datetime @@ -97,6 +98,8 @@ def main(): fileinfo = {} for file in args.files: fnmatch = re.match('([GR])[a-zA-Z]*_(.*)_4188x4110_(\d+)\.(\w+)', Path(file).name) + if not fnmatch: + fnmatch = re.match('([GR])[a-zA-Z]*_(.*)_(\d+)\.(\w+)', Path(file).name) if fnmatch: timestamp = datetime.datetime.fromtimestamp(Path(file).stat().st_mtime) fileinfo[file] = {'det': fnmatch.group(1), @@ -117,8 +120,9 @@ def main(): txtfiles = [file for file in fileinfo.keys() if fileinfo[file]['ext'] == 'txt'] num_colors = len(txtfiles) - colors = ['r', 'g', 'b', 'c', 'm', 'y', 'k'] - +# colors = ['b', 'g', 'r', 'c', 'm', 'y', 'k'] + colors = {'R': ['r', 'm', 'y'], + 'G': ['b', 'c', 'g']} plt.figure(figsize=(12,8)) for i,file in enumerate(txtfiles): print(f'Getting FFT of {file}') @@ -127,11 +131,13 @@ def main(): allnoise = np.std(alldata[200:1153])#[14:146]) # exclude the spike from parallel read out xf, yfft = do_fft(xs[100000:]*1e-9, ys[100000:]) label = f"{fileinfo[file]['det']} {fileinfo[file]['frameno']}: {fileinfo[file]['label']}" + color = colors[fileinfo[file].get('det')].pop(0) + print(color) multiplier = 1#e3**i - plt.loglog(xf, yfft*multiplier, f'{colors[i]}-', alpha=0.25) +# plt.loglog(xf, yfft*multiplier, f'{colors[i]}-', alpha=0.25) # Bin data and plot mean in each bin means, bins, binnumber = scipy.stats.binned_statistic(xf, yfft, statistic='mean', bins=10000) - plt.loglog(bins[1:], means*multiplier, f'{colors[i]}-', alpha=0.75, + plt.loglog(bins[1:], means*multiplier, f'{color}-', alpha=0.75, drawstyle='steps-pre', label=label) if args.marker: freq = args.marker*1e3 @@ -142,6 +148,7 @@ def main(): plt.legend() plt.xlabel("Freq (Hz)") plt.xlim(1e4,5e7) + plt.ylim(3e2,3e5) plt.ylabel('Amplitude') plt.grid() plt.savefig(args.outfile, bbox_inches='tight', pad_inches=0.1) diff --git a/kpf/engineering/plot_read_noise.py b/kpf/engineering/plot_read_noise.py index 96e67b921..e43acd58e 100644 --- a/kpf/engineering/plot_read_noise.py +++ b/kpf/engineering/plot_read_noise.py @@ -85,30 +85,31 @@ def main(): rotation='vertical') plt.ylabel('Read Noise (e-)') # plt.ylim(min(G_RN)*0.98, max(G_RN)*1.05) - plt.ylim(3.5, max(G_RN)+3) + plt.ylim(min(G_RN)-1, min([15, max(G_RN)+3])) plt.xticks(range(len(G_frameno)), G_frameno) plt.grid() - print('# Red Noise Measurements') - R_offset = 0.1 - plt.subplot(2,1,2) - R_RN = [fileinfo[file]['rn_e'] for file in redfiles] - R_RNos = [fileinfo[file]['rn_oscan'] for file in redfiles] - R_timestamp = [fileinfo[file]['timestamp'] for file in redfiles] - R_label = [fileinfo[file]['label'] for file in redfiles] - R_frameno = [fileinfo[file]['frameno'] for file in redfiles] - plt.plot(R_RN, 'ro') - plt.plot(R_RNos, 'ro', alpha=0.5) - for i,file in enumerate(redfiles): - print(f"{fileinfo[file]['timestr']} {file:50s}: {fileinfo[file]['rn_e']:5.2f} {fileinfo[file]['rn_oscan']:5.2f}") - plt.text(i, fileinfo[file]['rn_e']+R_offset, R_label[i], - rotation='vertical') - plt.ylabel('Read Noise (e-)') -# plt.ylim(min(R_RN)*0.98, max(R_RN)*1.05) - plt.ylim(1.5, max(R_RN)+1) - plt.xlabel('Frame Number') - plt.xticks(range(len(R_frameno)), R_frameno) - plt.grid() + if len(redfiles) > 0: + print('# Red Noise Measurements') + R_offset = 0.1 + plt.subplot(2,1,2) + R_RN = [fileinfo[file]['rn_e'] for file in redfiles] + R_RNos = [fileinfo[file]['rn_oscan'] for file in redfiles] + R_timestamp = [fileinfo[file]['timestamp'] for file in redfiles] + R_label = [fileinfo[file]['label'] for file in redfiles] + R_frameno = [fileinfo[file]['frameno'] for file in redfiles] + plt.plot(R_RN, 'ro') + plt.plot(R_RNos, 'ro', alpha=0.5) + for i,file in enumerate(redfiles): + print(f"{fileinfo[file]['timestr']} {file:50s}: {fileinfo[file]['rn_e']:5.2f} {fileinfo[file]['rn_oscan']:5.2f}") + plt.text(i, fileinfo[file]['rn_e']+R_offset, R_label[i], + rotation='vertical') + plt.ylabel('Read Noise (e-)') + # plt.ylim(min(R_RN)*0.98, max(R_RN)*1.05) + plt.ylim(min(R_RN)-0.5, min([9, max(R_RN)+1])) + plt.xlabel('Frame Number') + plt.xticks(range(len(R_frameno)), R_frameno) + plt.grid() plt.savefig('ReadNoise.png', bbox_inches='tight', pad_inches=0.1) plt.show() diff --git a/kpf/etc/ExpMeterBins.yaml b/kpf/etc/ExpMeterBins.yaml new file mode 100644 index 000000000..d4e845f1e --- /dev/null +++ b/kpf/etc/ExpMeterBins.yaml @@ -0,0 +1,5 @@ +'452': 1 +'548': 2 +'652': 3 +'747': 4 +'852': 4 diff --git a/kpf/etc/ExposureMeterZeroPoints.yaml b/kpf/etc/ExposureMeterZeroPoints.yaml new file mode 100644 index 000000000..c6156f465 --- /dev/null +++ b/kpf/etc/ExposureMeterZeroPoints.yaml @@ -0,0 +1,5 @@ +1: 21.296861833059204 +2: 21.802875631774594 +3: 21.73859924329517 +4: 21.385171098414354 +SUM: 23.06521519492087 diff --git a/kpf/etc/InstrumentZeroPoints.yaml b/kpf/etc/InstrumentZeroPoints.yaml new file mode 100644 index 000000000..0c2bba181 --- /dev/null +++ b/kpf/etc/InstrumentZeroPoints.yaml @@ -0,0 +1,5 @@ +'452': 13.664362089856091 +'548': 14.964690898633878 +'652': 15.329155098901115 +'747': 15.466755734095996 +'852': 15.073233551565435 diff --git a/kpf/etc/__init__.py b/kpf/etc/__init__.py new file mode 100644 index 000000000..c00fc1c66 --- /dev/null +++ b/kpf/etc/__init__.py @@ -0,0 +1,24 @@ +from pathlib import Path +import yaml + +'''Data files for the zero points were calculated by examining several years of +DRP outputs. See analysis in this repo: +https://github.com/joshwalawender/KPF-ZPETC/ +or on the KPF web page: +https://www2.keck.hawaii.edu/inst/kpf/sensitivity/ +under the "Zero Point Analysis" title. +''' + +ZPfile = Path(__file__).parent / 'InstrumentZeroPoints.yaml' +with open(ZPfile, 'r') as f: + ZeroPoint = yaml.safe_load(f) + + +EMZPfile = Path(__file__).parent / 'ExposureMeterZeroPoints.yaml' +with open(EMZPfile, 'r') as f: + EMZeroPoint = yaml.safe_load(f) + + +EMbinfile = Path(__file__).parent / 'ExpMeterBins.yaml' +with open(EMbinfile, 'r') as f: + best_EMbin = yaml.safe_load(f) diff --git a/kpf/etc/etc.py b/kpf/etc/etc.py new file mode 100644 index 000000000..dce968dfc --- /dev/null +++ b/kpf/etc/etc.py @@ -0,0 +1,103 @@ +from kpf import log, cfg +from kpf.exceptions import * +from kpf.KPFTranslatorFunction import KPFFunction, KPFScript +from kpf.etc import ZeroPoint, EMZeroPoint, best_EMbin + + +class ZPETC(KPFFunction): + '''Estimate the typical number of photons per reduced pixel near the + specified wavelength given the Gaia G magnitude of the target and the + exposure time. + + Args: + Gmag (float): The Gaia G magnitude of the target. + exptime (float): The exposure time in seconds. + wav (str): The science wavelength (in nm) as a string. One of: + '452', '548', '652', '747', '852'. + EMbin (int): The exposure meter bin (as an int). One of: 1, 2, 3, 4 + ''' + @classmethod + def pre_condition(cls, args): + check_input(args, 'Gmag', allowed_types=[float, int]) + check_input(args, 'exptime', allowed_types=[float, int], value_min=1) + check_input(args, 'wav', allowed_types=[str], + allowed_values=list(ZeroPoint.keys())) + + @classmethod + def perform(cls, args): + Gmag = args.get('Gmag') + exptime = args.get('exptime') + wav = args.get('wav') + photons = float(exptime*10**((Gmag - ZeroPoint[wav])/-2.5)) + if args.get('print'): + print(f"{photons:.0f}") + return photons + + @classmethod + def post_condition(cls, args): + pass + + @classmethod + def add_cmdline_args(cls, parser): + parser.add_argument('Gmag', type=float, + help="The Gaia G magnitude of the target.") + parser.add_argument('exptime', type=float, + help="The exposure time in seconds.") + parser.add_argument('wav', type=str, + choices=['452', '548', '652', '747', '852'], + help="The science wavelength (in nm) as a string.") + parser.add_argument('--print', '-p', dest="print", + default=False, action="store_true", + help='Print the result to screen?') + return super().add_cmdline_args(parser) + + +class EMADU(KPFFunction): + '''Estimate the number of Exposure Meter ADU in the given bin corresponding + to the specified number of science photons per reduced pixel near the + specified wavelength + + Args: + photons (float): The number of science photons per reduced pixel near + the specified wavelength. + wav (str): The science wavelength (in nm) as a string. One of: + '452', '548', '652', '747', '852'. + EMbin (int): The exposure meter bin (as an int). One of: 1, 2, 3, 4 + ''' + @classmethod + def pre_condition(cls, args): + check_input(args, 'photons', allowed_types=[float, int], value_min=1) + check_input(args, 'wav', allowed_types=[str], + allowed_values=list(ZeroPoint.keys())) + check_input(args, 'EMbin', allowed_types=[int], + allowed_values=list(EMZeroPoint.keys())) + + @classmethod + def perform(cls, args): + photons = args.get('photons') + wav = args.get('wav') + EMbin = args.get('EMbin') + ZPdiff = ZeroPoint[wav] - EMZeroPoint[EMbin] + ExpMeter_ADU = photons*10**(-ZPdiff/2.5) + if args.get('print'): + print(f"{ExpMeter_ADU:.0f}") + return ExpMeter_ADU + + @classmethod + def post_condition(cls, args): + pass + + @classmethod + def add_cmdline_args(cls, parser): + parser.add_argument('photons', type=float, + help="The number of science photons per reduced pixel near the specified wavelength.") + parser.add_argument('wav', type=str, + choices=['452', '548', '652', '747', '852'], + help="The science wavelength (in nm) as a string.") + parser.add_argument('EMbin', type=int, + choices=[1, 2, 3, 4], + help="The exposure meter bin as an int.") + parser.add_argument('--print', '-p', dest="print", + default=False, action="store_true", + help='Print the result to screen?') + return super().add_cmdline_args(parser) diff --git a/kpf/kpf_inst_config.ini b/kpf/kpf_inst_config.ini index efab67cfb..aec5510b6 100644 --- a/kpf/kpf_inst_config.ini +++ b/kpf/kpf_inst_config.ini @@ -1,3 +1,8 @@ +[operations] +lead_sa = jwalawender +deputy_sa = syeh +info_email = kpf_info + [telescope] telnr = 1 max_offset = 900 diff --git a/kpf/linking_table.yml b/kpf/linking_table.yml index cf50c00b7..5496ef368 100644 --- a/kpf/linking_table.yml +++ b/kpf/linking_table.yml @@ -24,6 +24,8 @@ links: cmd: ao.SetupAOforACAM.SetupAOforACAM SetupAOforKPF: cmd: ao.SetupAOforKPF.SetupAOforKPF + ShutdownAOforKPF: + cmd: ao.ShutdownAOforKPF.ShutdownAOforKPF TurnHepaOff: cmd: ao.TurnHepaOff.TurnHepaOff TurnHepaOn: @@ -87,10 +89,6 @@ links: cmd: engineering.TakeADCOffsetGridData.TakeADCOffsetGridData TakeGuiderSensitivityData: cmd: engineering.TakeGuiderSensitivityData.TakeGuiderSensitivityData - TriggerGreenMiniFill: - cmd: engineering.TriggerGreenMiniFill.TriggerGreenMiniFill - TriggerRedMiniFill: - cmd: engineering.TriggerRedMiniFill.TriggerRedMiniFill # Engineering/Analysis AnalyzeGridSearch: cmd: engineering.analysis.AnalyzeGridSearch.AnalyzeGridSearch @@ -100,6 +98,11 @@ links: cmd: engineering.analysis.Fit2DGridSearch.Fit2DGridSearch CountCameraErrors: cmd: engineering.analysis.CountCameraErrors.CountCameraErrors + # ETC + ZPETC: + cmd: etc.etc.ZPETC + EMADU: + cmd: etc.etc.EMADU # ExpMeter BuildMasterBias: cmd: expmeter.BuildMasterBias.BuildMasterBias @@ -333,6 +336,8 @@ links: cmd: utils.CheckAllowScheduledCals.CheckAllowScheduledCals EndOfNight: cmd: utils.EndOfNight.EndOfNight + ScheduleFilesCheck: + cmd: utils.ScheduleFilesCheck.ScheduleFilesCheck SetObserverFromSchedule: cmd: utils.SetObserverFromSchedule.SetObserverFromSchedule SetOutdirs: diff --git a/kpf/observatoryAPIs/GetObservingBlocks.py b/kpf/observatoryAPIs/GetObservingBlocks.py index 3fdc352c8..cac129bb6 100644 --- a/kpf/observatoryAPIs/GetObservingBlocks.py +++ b/kpf/observatoryAPIs/GetObservingBlocks.py @@ -18,8 +18,9 @@ def pre_condition(cls, args): @classmethod def perform(cls, args): + verbose = args.get('verbose', False) params = {'id': args.get('OBid', '')} - OBs, failure_messages = get_OBs_from_KPFCC_API(params) + OBs, failure_messages = get_OBs_from_KPFCC_API(params, verbose=verbose) if args.get('show_history', False): print(f'# Observing History for {OBs[0].summary()}') for i,h in enumerate(OBs[0].History): @@ -36,6 +37,16 @@ def perform(cls, args): if len(failure_messages) > 0: for msg in failure_messages: print(msg) + + # Print if verbose + if verbose: + print(f"# Failure Messages:") + print(failure_messages) + for i,OB in enumerate(OBs): + print(f"# Parsed OB {i+1}:") + print(OB.__repr__()) + + return OBs, failure_messages @classmethod @@ -49,6 +60,9 @@ def add_cmdline_args(cls, parser): parser.add_argument('--history', '--show_history', dest="show_history", default=False, action="store_true", help='Print history to screen?') + parser.add_argument('-v', '--verbose', dest="verbose", + default=False, action="store_true", + help='Show verbose OB information?') return super().add_cmdline_args(parser) diff --git a/kpf/observatoryAPIs/GetTelescopeRelease.py b/kpf/observatoryAPIs/GetTelescopeRelease.py index bb54056c1..5b156f056 100644 --- a/kpf/observatoryAPIs/GetTelescopeRelease.py +++ b/kpf/observatoryAPIs/GetTelescopeRelease.py @@ -25,7 +25,7 @@ def perform(cls, args): params = {'telnr': args.get('telnr', 1)} result = query_observatoryAPI('schedule', 'getTelescopeReadyState', params) log.debug(f'getTelescopeReadyState returned {result}') - return result.get('State', '') == 'Ready' + return result.get('State', '') in ['Ready', 'Foul Weather'] @classmethod def post_condition(cls, args): diff --git a/kpf/observatoryAPIs/__init__.py b/kpf/observatoryAPIs/__init__.py index 4cca8d7cb..dcebf1b13 100644 --- a/kpf/observatoryAPIs/__init__.py +++ b/kpf/observatoryAPIs/__init__.py @@ -137,7 +137,7 @@ def setKPFJunkValue(OBid, timestamp, junk=True): return query_observatoryAPI('proposal', 'setKPFJunkValue', params, post=True) -def get_OBs_from_KPFCC_API(params): +def get_OBs_from_KPFCC_API(params, verbose=False): result = query_observatoryAPI('proposal', 'getKPFObservingBlock', params) if result is None: return [] @@ -146,6 +146,28 @@ def get_OBs_from_KPFCC_API(params): n = len(result) for i,entry in enumerate(result): log.debug(f'Parsing entry {i+1} of {n}') + # Verbose Printing + if verbose: + print(f'# API Result {i+1} of {n}:') + OBentry = copy.deepcopy(entry) + target = OBentry.pop('Target', None) + if target is not None: + print(f'# Target:') + print(target) + observations = OBentry.pop('Observations', []) + for iobs,obs in enumerate(observations): + print(f'# Observation {iobs+1}:') + print(obs) + calibrations = OBentry.pop('Calibrations', []) + for ical,cal in enumerate(calibrations): + print(f'# Calibration {ical+1}:') + print(cal) + history = OBentry.pop('History', None) + for key in OBentry.keys(): + print(f"# {key}: {OBentry[key]}") + if history is not None: + print(f'# History:') + print(history) if not isinstance(entry, dict): failures.append([f'{entry}', 'Entry is not dict']) else: diff --git a/kpf/scripts/RunOB.py b/kpf/scripts/RunOB.py index f7d5f6702..e8a75564a 100644 --- a/kpf/scripts/RunOB.py +++ b/kpf/scripts/RunOB.py @@ -219,12 +219,12 @@ def perform(cls, args, OB=None): log.info(msg) SCRIPTMSG.write(msg) print() - print("###############################################################") - print(" WARNING: The telescope control system target name: {TARGNAME}") - print(" WARNING: does not match the OB target name: {OB.Target.TargetName}") + print(f"###############################################################") + print(f" WARNING: The telescope control system target name: {TARGNAME}") + print(f" WARNING: does not match the OB target name: {OB.Target.TargetName}") print() - print(" Press 'Enter' to begin exposure(s) anyway or 'a' to abort script") - print("###############################################################") + print(f" Press 'Enter' to begin exposure(s) anyway or 'a' to abort script") + print(f"###############################################################") print() user_input = input() log.debug(f'response: "{user_input}"') diff --git a/kpf/scripts/RunSoCal.py b/kpf/scripts/RunSoCal.py index 0f351d7b6..4a715f92c 100644 --- a/kpf/scripts/RunSoCal.py +++ b/kpf/scripts/RunSoCal.py @@ -148,6 +148,7 @@ def perform(cls, args, OB=None): time.sleep(wait) elif now_decimal > end_time: log.info("End time for today's SoCal window has passed") + clear_script_keywords() return ConfigureFIU.execute({'mode': 'Calibration'}) diff --git a/kpf/socal/IsSoCalShutDown.py b/kpf/socal/IsSoCalShutDown.py index f23504265..f42b5a759 100644 --- a/kpf/socal/IsSoCalShutDown.py +++ b/kpf/socal/IsSoCalShutDown.py @@ -55,5 +55,4 @@ def add_cmdline_args(cls, parser): parser.add_argument('--email', dest="email", default=False, action="store_true", help='Send email if SoCal is not shut down?') - return super().add_cmdline_args(parser) diff --git a/kpf/socal/WaitForSoCalOnTarget.py b/kpf/socal/WaitForSoCalOnTarget.py index 46ae63620..fca7bbdb0 100644 --- a/kpf/socal/WaitForSoCalOnTarget.py +++ b/kpf/socal/WaitForSoCalOnTarget.py @@ -30,16 +30,20 @@ def pre_condition(cls, args): def perform(cls, args): timeout = cfg.getfloat('SoCal', 'enclosure_status_time', fallback=10) pyrirrad_threshold = cfg.getfloat('SoCal', 'pyrirrad_threshold', fallback=1000) - expr = '($kpfsocal.ENCSTA == 0) ' - expr += 'and ($kpfsocal.EKOONLINE == Online)' - expr += 'and ($kpfsocal.EKOMODE == 3)' - expr += f'and ($kpfsocal.PYRIRRAD > {pyrirrad_threshold})' - expr += 'and ($kpfsocal.AUTONOMOUS == 1)' - expr += 'and ($kpfsocal.CAN_OPEN == True)' - expr += 'and ($kpfsocal.IS_OPEN == True)' - expr += 'and ($kpfsocal.IS_TRACKING == True)' - expr += 'and ($kpfsocal.ONLINE == True)' - expr += 'and ($kpfsocal.STATE == Tracking)' + # EKO Tracker Status + expr = '($kpfsocal.EKOONLINE == Online) ' + expr += 'and ($kpfsocal.EKOMODE == 3) ' + # Pyrheliometer threshold + expr += f'and ($kpfsocal.PYRIRRAD > {pyrirrad_threshold}) ' + # Enclosure Status +# expr += 'and ($kpfsocal.ENCSTA == 0) ' + # Sequencer Status +# expr += 'and ($kpfsocal.AUTONOMOUS == 1) ' +# expr += 'and ($kpfsocal.CAN_OPEN == True) ' +# expr += 'and ($kpfsocal.IS_OPEN == True) ' +# expr += 'and ($kpfsocal.IS_TRACKING == True) ' +# expr += 'and ($kpfsocal.ONLINE == True) ' +# expr += 'and ($kpfsocal.STATE == Tracking) ' on_target = ktl.waitFor(expr, timeout=timeout) msg = {True: 'On Target', False: 'NOT On Target'}[on_target] print(msg) diff --git a/kpf/utils/BuildCalOB.py b/kpf/utils/BuildCalOB.py index d6230af33..c9013762d 100644 --- a/kpf/utils/BuildCalOB.py +++ b/kpf/utils/BuildCalOB.py @@ -106,7 +106,9 @@ def perform(cls, args): print(f"# {cal_string}") duration = EstimateOBDuration.execute({'verbose': True}, OB=OB) if args.get('execute', False): - RunOB.execute({}, OB=OB) + runOBargs = {'waitforscript': args.get('waitforscript', False), + 'scheduled': args.get('scheduled', False)} + RunOB.execute(runOBargs, OB=OB) return OB @@ -117,16 +119,22 @@ def post_condition(cls, args): @classmethod def add_cmdline_args(cls, parser): parser.add_argument('calinputs', nargs='*', - help="Calibrations to take in the form ") + help="Calibrations to take in the form ") parser.add_argument("-v", "-t", "--time", "--estimate", dest="estimate", - default=False, action="store_true", - help="Estimate the execution time for this OB?") + default=False, action="store_true", + help="Estimate the execution time for this OB?") parser.add_argument("-s", "--save", dest="save", type=str, default='', - help="Save resulting OB to the specified file.") + help="Save resulting OB to the specified file.") parser.add_argument("-o", "--overwrite", dest="overwrite", - default=False, action="store_true", - help="Overwrite output file if it exists?") - parser.add_argument("--execute", dest="execute", - default=False, action="store_true", - help="Execute the resulting OB?") + default=False, action="store_true", + help="Overwrite output file if it exists?") + parser.add_argument("-x", "--execute", dest="execute", + default=False, action="store_true", + help="Execute the resulting OB?") + parser.add_argument('-w', '--waitforscript', dest="waitforscript", + default=False, action="store_true", + help='Wait for running script to end before starting?') + parser.add_argument('-a', '--scheduled', dest="scheduled", + default=False, action="store_true", + help='Script is scheduled and should obey ALLOWSCHEDULEDCALS keyword') return super().add_cmdline_args(parser) diff --git a/kpf/utils/EndOfNight.py b/kpf/utils/EndOfNight.py index e7e90fd78..82eb8ee65 100644 --- a/kpf/utils/EndOfNight.py +++ b/kpf/utils/EndOfNight.py @@ -8,9 +8,7 @@ from kpf.KPFTranslatorFunction import KPFFunction, KPFScript from kpf.scripts import (register_script, obey_scriptrun, check_scriptstop, add_script_log) -from kpf.ao.ControlAOHatch import ControlAOHatch -from kpf.ao.TurnHepaOn import TurnHepaOn -from kpf.ao.SendPCUtoHome import SendPCUtoHome +from kpf.ao.ShutdownAOforKPF import ShutdownAOforKPF from kpf.fiu.ShutdownTipTilt import ShutdownTipTilt from kpf.fiu.ConfigureFIU import ConfigureFIU from kpf.fiu.WaitForConfigureFIU import WaitForConfigureFIU @@ -159,22 +157,7 @@ def perform(cls, args): user_input = input() if user_input.lower() in ['y', 'yes', '']: log.debug('User chose to shut down AO') - log.info('Closing AO Hatch') - try: - ControlAOHatch.execute({'destination': 'closed'}) - except FailedToReachDestination: - log.error(f"AO hatch did not move successfully") - except Exception as e: - log.error(f"Failure controlling AO hatch") - log.error(e) - log.info('Sending PCU stage to Home position') - try: - SendPCUtoHome.execute({}) - except Exception as e: - log.error(f"Failure sending PCU to home") - log.error(e) - # log.info('Turning on AO HEPA Filter System') - # TurnHepaOn.execute({}) + ShutdownAOforKPF.execute({}) else: log.warning(f'User chose to skip AO shutdown') diff --git a/kpf/utils/ScheduleFilesCheck.py b/kpf/utils/ScheduleFilesCheck.py new file mode 100644 index 000000000..4e39f7b2d --- /dev/null +++ b/kpf/utils/ScheduleFilesCheck.py @@ -0,0 +1,127 @@ +from pathlib import Path +import datetime + +from kpf import log, cfg +from kpf.exceptions import * +from kpf.KPFTranslatorFunction import KPFFunction, KPFScript +from kpf.observatoryAPIs import get_semester_dates, query_observatoryAPI +from kpf.utils.SendEmail import SendEmail + + +##----------------------------------------------------------------------------- +## ScheduleFilesCheck +##----------------------------------------------------------------------------- +class ScheduleFilesCheck(KPFFunction): + '''Check whether all expected schedule files are present and count the + number of lines in each file and see if it is larger than 1. + + Args: + email (bool): Send an email if a problem is detected? + + Functions Called: + + - `kpf.observatoryAPIs.GetScheduledPrograms` + - `kpf.utils.SendEmail` + ''' + @classmethod + def pre_condition(cls, args): + pass + + @classmethod + def perform(cls, args): + errors = [] + utnow = datetime.datetime.utcnow() + date_string = utnow.strftime('%Y-%m-%d') + log.info(f"# Checking for schedule for the night of {date_string} HST") + semester, s_start, s_end = get_semester_dates(utnow) + + log.info(f'# KPF-CC Schedule File Check') + band_names = ['full-band1', 'full-band2', 'full-band3'] + + params = {'date': date_string, 'numdays': 1, 'telnr': 1, 'instrument': 'KPF'} + all_programs = query_observatoryAPI('schedule', 'getSchedule', params) + classical = [p for p in all_programs if p['Instrument'] == 'KPF'] + cadence = [p for p in all_programs if p['Instrument'] == 'KPF-CC'] + cadence_projects = [p['ProjCode'] for p in cadence] + + if len(cadence_projects) > 0: + log.info(f"# Found cadence programs: {cadence_projects}") + band_names.extend(['band1', 'band2', 'band3']) + + log.info(f"# Checking for {len(band_names)} schedules: {band_names}") + base_path = Path('/s/sdata1701/Schedules') + semester_path = base_path / semester + + date_path = semester_path / date_string + if date_path.exists() is False: + err = f"{str(date_path)} does not exist" + errors.append(err) + log.error(err) + + band_paths = [date_path / band for band in band_names] + line_counts = {} + for band_path in band_paths: + if band_path.exists() is False: + err = f"{str(band_path)} does not exist" + errors.append(err) + log.error(err) + output_path = band_path / 'output' + if output_path.exists() is False: + err = f"{str(output_path)} does not exist" + errors.append(err) + log.error(err) + output_file = output_path / 'night_plan.csv' + if output_file.exists() is False: + err = f"{str(output_file)} does not exist" + errors.append(err) + log.error(err) + try: + with open(output_file, 'r') as f: + lines = f.readlines() + nlines = len(lines) + line_counts[band_path.name] = nlines + if nlines <= 1: + err = f"{str(output_file)} has only {nlines} lines" + errors.append(err) + log.error(err) + except: + err = f'Failed to read {str(output_file)}' + errors.append(err) + log.error(err) + + # Results + log.info(f'# {str(date_path)}') + result_str = 'Band Line Count' + log.info(f'# {result_str}') + result_str += '\n' + for band in sorted(line_counts.keys()): + newline = f'{band:11s} {line_counts[band]:d}' + if line_counts[band] <= 1: + newline += ' <-- Low target count!' + log.info(f'# {newline}') + result_str += f"{newline}\n" + + # Send Email + if len(errors) > 0: + msg = 'KPF-CC Schedule May Be Bad\n\n' + if args.get('email', False) == True: + try: + to_value = cfg.get('operations', 'lead_sa', 'jwalawender') + SendEmail.execute({'Subject': f'KPF-CC Schedule May Be Bad', + 'Message': msg+result_str, + 'To': f'{to_value}@keck.hawaii.edu'}) + except Exception as email_err: + log.error(f'Sending email failed') + log.error(email_err) + + + @classmethod + def post_condition(cls, args): + pass + + @classmethod + def add_cmdline_args(cls, parser): + parser.add_argument('--email', dest="email", + default=False, action="store_true", + help='Send email if SoCal is not shut down?') + return super().add_cmdline_args(parser) diff --git a/kpf/utils/SendEmail.py b/kpf/utils/SendEmail.py index 4d654da77..1be0194d9 100644 --- a/kpf/utils/SendEmail.py +++ b/kpf/utils/SendEmail.py @@ -24,9 +24,10 @@ def pre_condition(cls, args): @classmethod def perform(cls, args): + default_to = cfg.get('operations', 'info_email', 'kpf_info') msg = MIMEText(args.get('Message', 'Test email. Please ignore.')) msg['To'] = args.get('To', 'kpf_info@keck.hawaii.edu') - msg['From'] = args.get('From', 'kpf_info@keck.hawaii.edu') + msg['From'] = args.get('From', f'{default_to}@keck.hawaii.edu') msg['Subject'] = args.get('Subject', 'KPF Alert') log.warning(f"Sending email, To {msg.get('To')}") log.warning(f"Sending email, Subject {msg.get('Subject')}") diff --git a/mkdocs.yml b/mkdocs.yml index 0f9f8ed5d..3b4f6df80 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -99,6 +99,7 @@ nav: - "SetAORotatorManual": scripts/SetAORotatorManual.md - "SetupAOforACAM": scripts/SetupAOforACAM.md - "SetupAOforKPF": scripts/SetupAOforKPF.md + - "ShutdownAOforKPF": scripts/ShutdownAOforKPF.md - "TurnHepaOff": scripts/TurnHepaOff.md - "TurnHepaOn": scripts/TurnHepaOn.md - "TurnLightSourceOff": scripts/TurnLightSourceOff.md @@ -107,6 +108,7 @@ nav: - "kpf.calbench": - "CalLampPower": scripts/CalLampPower.md - "IsCalSourceEnabled": scripts/IsCalSourceEnabled.md + - "ListLFCRuns": scripts/ListLFCRuns.md - "PredictNDFilters": scripts/PredictNDFilters.md - "SetCalSource": scripts/SetCalSource.md - "SetFlatFieldFiberPos": scripts/SetFlatFieldFiberPos.md @@ -131,8 +133,6 @@ nav: - "TakeADCGridData": scripts/TakeADCGridData.md - "TakeADCOffsetGridData": scripts/TakeADCOffsetGridData.md - "TakeGuiderSensitivityData": scripts/TakeGuiderSensitivityData.md - - "TriggerGreenMiniFill": scripts/TriggerGreenMiniFill.md - - "TriggerRedMiniFill": scripts/TriggerRedMiniFill.md - "AnalyzeGridSearch": scripts/AnalyzeGridSearch.md - "AnalyzeTipTiltPerformance": scripts/AnalyzeTipTiltPerformance.md - "Fit2DGridSearch": scripts/Fit2DGridSearch.md @@ -198,6 +198,7 @@ nav: - "SelectTarget": scripts/SelectTarget.md - "SetTargetList": scripts/SetTargetList.md - "kpf.observatoryAPIs": + - "GetCurrentScheduledProgram": scripts/GetCurrentScheduledProgram.md - "GetExecutionHistory": scripts/GetExecutionHistory.md - "GetKPFCCObservingBlocks": scripts/GetKPFCCObservingBlocks.md - "GetObservingBlocks": scripts/GetObservingBlocks.md @@ -258,6 +259,7 @@ nav: - "BuildCalOB": scripts/BuildCalOB.md - "CheckAllowScheduledCals": scripts/CheckAllowScheduledCals.md - "EndOfNight": scripts/EndOfNight.md + - "ScheduleFilesCheck": scripts/ScheduleFilesCheck.md - "SetObserverFromSchedule": scripts/SetObserverFromSchedule.md - "SetOutdirs": scripts/SetOutdirs.md - "StartGUIs": scripts/StartGUIs.md