""" Readers for Kislovodsk solar data files. Self-contained copy of selected functions from kmas.io. Nothing is imported from kmas. Helpers that those readers call (parse_filename, fix_contour_breaks and the SynMap class) are defined in this module. """ import copy import re from datetime import datetime from pathlib import Path import dateutil.parser as dparser import numpy as np import pandas as pd from astropy.io import fits from astropy.time import Time from skimage.transform import resize from sunpy.coordinates.sun import B0, L0, carrington_rotation_time def _parse_datetime(d_t, fuzzy=True): """ Parse a datetime string as naive UTC. Strips unknown Apache-style timezone suffixes (e.g. 'GG') and ignores tzinfo. """ if isinstance(d_t, str): d_t = re.sub(r'\s+[A-Z]{2,4}$', '', d_t.strip()) return dparser.parse(d_t, fuzzy=fuzzy, ignoretz=True) return d_t def _replace_dt_format(string): return string.replace('%Y', r'\d{4}').replace('%y', r'\d{2}').replace('%m', r'\d{2}').\ replace('%d', r'\d{2}').replace('%H', r'\d{2}').replace('%M', r'\d{2}').replace('%S', r'\d{2}') def parse_filename(path, dt_mask=None, use_regex=False): """ Parse the datetime from the file name. Parameters ---------- path : str or pathlib.Path Path to file. dt_mask: str, optional Datetime format for file name parsing. For example: 'xxx_%Y%m%d_%H%M%S' to parsing 'xxx_20000101_000000'. If dt_mask is None, fuzzy method will be applied. The default is None. use_regex: bool, optional Use regex to search dt_mask in file name. The default is None. Raises ------ ValueError No matches found for regex if the regex is not None. Returns ------- dt : datetime.datetime Date and time. """ stem = Path(path).stem.translate({ord(i): None for i in 'apm'}) if dt_mask is None: stem = stem.replace('.', '_') d_t = _parse_datetime(stem) else: if use_regex: regex = _replace_dt_format(dt_mask) match = re.search(regex, stem) if match is None: raise ValueError('The regular expression {} was not found in the file name.' .format(regex)) d_t = _parse_datetime(match.group(0)) else: d_t = datetime.strptime(stem, dt_mask) return d_t def fix_contour_breaks(contour): """ Correct breaks in the contour given in heliographic coordinates. Breaks that occur when crossing zero or 360 degrees of longitude. Parameters ---------- contour : numpy.ndarray Longitude and latitude coordinates, degrees: [[lon_0, lat_0], [lon_1, lat_1], ...]. Returns ------- contour : numpy.ndarray Longitude and latitude coordinates, degrees: [[lon_0, lat_0], [lon_1, lat_1], ...]. """ if (np.abs(np.diff(contour[:, 0])) < 180).all(): return contour cnt = contour.copy() for j in range(cnt.shape[0] - 1): diff = cnt[:, 0][j + 1] - cnt[:, 0][j] if diff > 180: cnt[:, 0][j + 1] = cnt[:, 0][j + 1] - 360 elif diff < -180: cnt[:, 0][j + 1] = cnt[:, 0][j + 1] + 360 return cnt def open_stop_txt(path): """ Open STOP *H.txt file as SolarImage. Parameters path : str or pathlib.Path Path to txt-file. to_image : bool, optional True if you need to convert the table to an image. The default is True. Returns ------- img : SolarImage Corresponding SolarImage object containing two layers: map of magnetic fields and map of sigma. df : pd.DataFrame The STOP data from the txt file table. Contains additional parameters in df.attrs. """ # Reading the header attrs = {'R': None, 'APN': None} with open(path, 'r', encoding='windows-1251') as file: for _ in range(20): line = file.readline() for key in attrs: if line.startswith(key): attrs[key] = float(line.split()[2]) # Reading the data df = pd.read_csv(path, encoding='windows-1251', skiprows=21, sep=r'\s+', engine='python', parse_dates=[['дата', 'время']], dayfirst=True) df.attrs = attrs return df def open_bp(path, ignore_footer=False): """ Open Kislovodsk bp-file. Parameters ---------- path : str or pathlib.Path Path to bp-file. ignore_footer : bool, optional Don't try to read the footer. The default is False. Returns ------- pandas.DataFrame A data frame with bp data. """ # Reading strings with iteration of encodings: encodings_to_try = ['cp1251', 'utf-8', 'koi8-r', 'iso-8859-5', 'cp866'] for encoding in encodings_to_try: try: with open(path, 'r', encoding=encoding) as f: lines = f.readlines() break except UnicodeDecodeError: continue else: # If all the encodings don't fit, we use utf-8 with error handling: print("Warning: Using utf-8 with error handling") with open(path, 'r', encoding='utf-8', errors='replace') as f: lines = f.readlines() # We read the column name, data, and footer: columns = [] data_rows = [] footer_lines = [] in_footer = False for line in lines: line = line.strip() if not line: continue # Determining the end of the file and the footer: if line.startswith('Total') or in_footer: if ignore_footer: continue in_footer = True footer_lines.append(line) continue # Reading the column names: if '|' in line and not columns: columns = [col.strip() for col in line.split('|') if col.strip()] continue # Reading the data: if columns: parts = [part for part in line.split() if part] if parts: # Moving the umbra label to the end of the line: if 'um' in parts: um_count = parts.count('um') parts = [part for part in parts if part != 'um'] parts.extend(['um'] * um_count) data_rows.append(parts) # Creating DataFrame, note that the number of data and header columns may vary: if not columns and data_rows: max_cols = max(len(row) for row in data_rows) if data_rows else 0 columns = [f'col_{i}' for i in range(max_cols)] if data_rows: df = pd.DataFrame(data_rows) # Equalize the number of columns if df.shape[1] > len(columns): extended_columns = columns + [f'extra_{j}' for j in range(len(columns), df.shape[1])] df.columns = extended_columns elif df.shape[1] < len(columns): df.columns = columns[:df.shape[1]] else: df.columns = columns # Convert numeric columns for col in df.columns: try: df[col] = pd.to_numeric(df[col]) except ValueError: pass else: df = pd.DataFrame() if footer_lines: df.attrs['footer'] = footer_lines return df def open_abp(path, calc_ephemeris=True, calc_center=False, **kwargs): """ Open Kislovodsk abp-file. Parameters ---------- path : str or pathlib.Path Path to abp-file. calc_ephemeris : bool, optional True for calculating solar ephemerides: B0 and L0 by date and time in the header of the abp file. The defalut is True. calc_center : bool, optional True to calculate the center point of each object. The default is False. kwargs Keyword arguments passed to parse_filename(): dt_mask: str, optional Datetime format for file name parsing. For example: 'xxx_%Y%m%d_%H%M%S' to parsing 'xxx_20000101_000000'. If dt_mask is None, fuzzy method will be applied. The default is None. use_regex: bool, optional Use regex to search dt_mask in file name. The default is None. Returns ------- df : pandas.DataFrame A data frame with abp data, сontaining pairs of coordinates [x, y]. Contains additional parameters (disk center, radius, ...) in df.attrs. """ with open(path, 'r') as file: lines = file.readlines() line_0 = lines[0].split() attrs = { 'center': (int(line_0[0]), int(line_0[1])), 'R': int(line_0[2]), 'P': float(line_0[3]), 'B0': float(line_0[4]), 'L0': float(line_0[5]), } try: attrs['date_time'] = parse_filename(path, **kwargs) if calc_ephemeris: attrs['B0'] = B0(attrs['date_time']).degree attrs['L0'] = L0(attrs['date_time']).degree except dparser.ParserError: pass n_skip = int(lines[1].split()[0]) n_objects = (len(lines) - 3 - n_skip) // 2 info = np.array([lines[3 + n_skip + 2 * i].split() for i in range(n_objects)]).astype(int) data = [np.array(lines[4 + n_skip + 2 * i].split()).astype(int) for i in range(n_objects)] # Filling in the data frame df = pd.DataFrame(columns=['object_number', 'group_number', 'umbra', 'id', 'area_px', 'points']) if n_objects: df['object_number'] = info[:, 0] df['area_px'] = info[:, 1] df['group_number'] = info[:, 2] df['umbra'] = info[:, -1].astype(bool) # Create a sunspot ID: idn = np.char.add(info[:, 2].astype(str), '-') idn = np.char.add(idn, info[:, 5].astype(str)) df['id'] = np.where(info[:, 6] == 0, idn, np.char.add(np.char.add(idn, '-'), info[:, 6].astype(str))) # If the number of digits in the string is not a multiple of three # (the file is corrupted), the array is truncated df['points'] = [arr[:arr.shape[0] // 3 * 3].reshape((-1, 3))[:, [0, 1]] for arr in data] df = df[df['area_px'] > 0] df = df.set_index('object_number') if calc_center: for i, row in df.iterrows(): df.at[i, 'x'] = np.mean(row['points'][:, 0]) df.at[i, 'y'] = np.mean(row['points'][:, 1]) df.attrs = attrs return df def open_grp(path): """ Read Kislovodsk grp-file. Parameters ---------- path : str or pathlib.Path Path to grp-file. Returns ------- df : pandas.DataFrame A data frame with sunspots data. Contains additional parameters (date and time, observer name, ...) in df.attrs. """ with open(path, 'r', encoding='utf-8', errors='ignore') as file: lines = file.readlines() if len(lines) < 3: return None, None # Read the file header information: line_1 = lines[1].split() match = re.search(r'\d{4}/\d{2}/\d{2} \d{2}:\d{2}', lines[0]) dt = datetime.strptime(match.group(), '%Y/%m/%d %H:%M') if match else None match = re.search(r'Observer:\s*([A-Za-z]+(?:\s+[A-Za-z.]+)*)\s*$', lines[0]) observer = match.group(1).strip() if match else '' attrs = { 'date_time': dt, 'Observer' : observer, 'L0' : float(line_1[9]), 'B0' : float(line_1[7]), 'P' : float(line_1[12][1:-1]), 'P_d' : float(line_1[11]), } # Defining positions of table column separators: seps = [s.start() for s in re.finditer(r'\|', lines[2])] seps[0] += 1 seps.append(seps[-1] + 6) # Reading column names: keys = [s.strip() for s in lines[2].split('|')] # Initializing a dataframe: df = pd.DataFrame(columns=['id', 'umbra'] + keys + ['lat_gr', 'lon_gr', 'Wg']) i = 3 n_group = None w_group = 0 n_spot = 0 n_umbra = 0 row = 0 while i < len(lines): # Cycle through the rows of the dataframe line = lines[i].lstrip() if len(line) < 5: i += 1 continue if line.startswith('Total'): # Reached the end of the file attrs['W'] = int(line.split('W:')[1]) break if line.startswith('Ngr'): # New group data block reached fields = lines[i].split() n_group = int(fields[1]) # Current group number w_group = int(fields[-1]) # Current group Wolf number # Average latitude and longitude of the current group lat_group, lon_group = lines[i + 1].rstrip().split('qs=')[-1].split('fis=') i += 2 continue # Adding current group parameters: number, Wolf and position: df.at[row, 'N'] = n_group df.at[row, 'Wg'] = w_group df.at[row, 'lon_gr'] = float(lon_group) df.at[row, 'lat_gr'] = float(lat_group) field = lines[i][:seps[0]] # Conutrating the individual spot number: if not 'um' in field: # for penumbra n_spot = int(field) df.at[row, 'umbra'] = False df.at[row, 'id'] = f'{n_group}-{n_spot}' else: # for umbra n_umbra = int(field[2:].strip()) df.at[row, 'umbra'] = True df.at[row, 'id'] = f'{n_group}-{n_spot}-{n_umbra}' # Reading all data from the table: try: for j in range(1, len(seps)): df.at[row, keys[j]] = float(lines[i][seps[j-1]: seps[j]]) except ValueError: line = lines[i].split() for j in range(1, len(seps)): df.at[row, keys[j]] = float(line[j]) row += 1 i += 1 df['Ind'] = df['Ind'].astype(int) df.attrs = attrs return df def open_pral(path, calc_ephemeris=True, calc_center=False, **kwargs): """ Open Kislovodsk pral-file containing data on prominences. Parameters ---------- path : str or pathlib.Path Path to abp-file. calc_ephemeris : bool, optional True for calculating solar ephemerides: B0 and L0 by date and time in the header of the abp file. The defalut is True. calc_center : bool, optional True to calculate the center point of each object. The default is False. kwargs Keyword arguments passed to parse_filename(): dt_mask: str, optional Datetime format for file name parsing. For example: 'xxx_%Y%m%d_%H%M%S' to parsing 'xxx_20000101_000000'. If dt_mask is None, fuzzy method will be applied. The default is None. use_regex: bool, optional Use regex to search dt_mask in file name. The default is None. Returns ------- df : pandas.DataFrame A data frame with abp data, сontaining pairs of radial and angular coordinates [r, a]. Contains additional parameters (disk center, radius, ...) in df.attrs. """ with open(path, 'r') as file: lines = file.readlines() line_0 = lines[0].split() attrs = { 'center': (int(line_0[0]), int(line_0[1])), 'R': int(line_0[2]), 'P': float(line_0[3]), 'B0': float(line_0[4]), 'L0': float(line_0[5]), } try: attrs['date_time'] = parse_filename(path, **kwargs) if calc_ephemeris: attrs['B0'] = B0(attrs['date_time']).degree attrs['L0'] = L0(attrs['date_time']).degree except dparser.ParserError: pass n_objects = (len(lines) - 2) // 2 info = np.array([lines[2 + 2 * i].split() for i in range(n_objects)]).astype(np.float32) data = [np.array(lines[3 + 2 * i].split()).astype(np.float32) for i in range(n_objects)] # Filling in the data frame df = pd.DataFrame(columns=['object_number', 'area_px', 'r_min', 'r_max', 'a_min', 'a_max', 'points']) if n_objects: df['object_number'] = info[:, 0].astype(int) df['area_px'] = info[:, 1].astype(int) df['r_min'] = info[:, 4] df['r_max'] = info[:, 5] df['a_min'] = info[:, 6] df['a_max'] = info[:, 7] df['points'] = [arr.reshape((-1, 3))[:, [0, 1]] for arr in data] df = df[df['area_px'] > 0] df = df.set_index('object_number') if calc_center: for i, row in df.iterrows(): df.at[i, 'r'] = np.mean(row['points'][:, 0]) df.at[i, 'a'] = np.mean(row['points'][:, 1]) df.attrs = attrs return df def open_cnt(path, fix_breaks=False, remove_dups=False, **kwargs): """ Open Kislovodsk cnt-file. Parameters ---------- path : str or pathlib.Path Path to abp-file. fix_breaks : bool, optional Fix the contour breaks in the central meridian. The default is False. remove_dups : bool, optional Remove duplicate points. The default is False. kwargs Keyword arguments passed to parse_filename(): dt_mask: str, optional Datetime format for file name parsing. For example: 'xxx_%Y%m%d_%H%M%S' to parsing 'xxx_20000101_000000'. If dt_mask is None, fuzzy method will be applied. The default is None. use_regex: bool, optional Use regex to search dt_mask in file name. The default is None. Returns ------- df : pandas.DataFrame A data frame with cnt data, сontaining pairs of coordinates [longitude, latitude]. Contains additional parameters (ephemeris, date and time, ...) in df.attrs. """ with open(path, 'r') as file: lines = file.readlines() line_0 = lines[0].split() attrs = { 'P': float(line_0[3]), 'B0': float(line_0[4]), 'L0': float(line_0[5]), } try: attrs['date_time'] = parse_filename(path, **kwargs) except dparser.ParserError: pass n_objects = (len(lines) - 2) // 2 contours = [] for i in range(n_objects): cnt = np.array(lines[3 + 2 * i].split()).astype(np.float32).reshape((-1, 2))[:, [1, 0]] if fix_breaks: cnt = fix_contour_breaks(cnt) if remove_dups: _, idx = np.unique(cnt, axis=0, return_index=True) cnt = cnt[np.sort(idx, axis=0)] contours.append(cnt) df = pd.DataFrame(data={'contour': contours}) df.attrs = attrs return df def open_corona(path): """ Open Kislovodsk corona data file: gck*.dat or rck*.dat. Parameters ---------- path : str or pathlib.Path Path to dat-file. Returns ------- df : pandas.DataFrame A data frame with corona data. Columns - clockwise angle from the north pole, in degrees. Intensity -1 corresponds to empty values. df.attrs contains wavelength: 5303 А (green corona) or 6374 A (red corona). Notes ----- The angle grid is taken from the first line that contains ``T.U.``; if that line is missing or unreadable, 0..355° with a 5° step is used (Lyot coronagraph). Month labels in KMAS exports are often corrupted (Cyrillic lookalikes, etc.). Dedicated short label lines are therefore treated as ordered month separators: try to recognize the name, otherwise advance to the next calendar month. January data comes before the first such label; later labels are Feb, Mar, ... """ months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'] def _ascii_letters(text): return re.sub(r'[^a-zA-Z]', '', text).lower() def _match_month_name(token): """Recognize a month from Latin letters only; ignore unknown glyphs.""" ascii_token = _ascii_letters(token) if not ascii_token: return None for i, name in enumerate(months): if ascii_token == name or name in token.lower(): return i + 1 # Unique subsequence match: "Mr" -> mar, "pr" -> apr, "Nv" -> nov. if len(ascii_token) >= 2: hits = [] for i, name in enumerate(months): it = iter(name) if all(ch in it for ch in ascii_token): hits.append(i + 1) if len(hits) == 1: return hits[0] return None def _is_month_label_line(text): # Short non-data separator between monthly blocks. Labels may be Latin, # Cyrillic lookalikes, or otherwise corrupted — do not require readable text. s = text.strip() if not s or len(s) > 15 or re.match(r'^\d', s): return False return not re.search(r'\d{2,}', s) def _trailing_month_token(text): match = re.search(r'([A-Za-zА-Яа-яЁё]{3,})\s*$', text.strip()) return match.group(1) if match else None def _parse_int_row(text): # Missing values are "-", "x" or "?"; glued month labels must be ignored. cleaned = re.sub(r'(?= 2: return np.array(angles, dtype=int) return default_angles.copy() path = Path(path) with open(path, 'r', encoding='utf-8', errors='ignore') as file: lines = file.readlines() wavelength = int(lines[0].split()[-1]) try: year = int(lines[1].split()[-1]) except ValueError: match = re.search(r'((?:19|20)\d{2})', path.stem, re.IGNORECASE) if not match: raise ValueError(f'Cannot determine year from corona file header or name: {path}') year = int(match.group(1)) dt = datetime(year, 1, 1) header_idx = next((i for i, line in enumerate(lines) if 'T.U' in line), None) if header_idx is None: angles = default_angles.copy() body_start = 3 else: angles = _parse_angle_header(lines[header_idx]) body_start = header_idx + 1 df = pd.DataFrame(index=pd.to_datetime([]), columns=angles) for line in lines[body_start:]: if 'T.U' in line: break values = _parse_int_row(line) if (len(line) > 100 or re.match(r'\s*\d', line)) else np.array([], dtype=int) if values.size >= 3: try: dt = dt.replace(day=int(values[0]), hour=int(values[1]), minute=int(values[2])) except ValueError: continue intensity = values[3:] if intensity.shape[0] < df.shape[1]: row = -np.ones(df.shape[1]) row[:intensity.shape[0]] = intensity elif intensity.shape[0] > df.shape[1]: row = intensity[:df.shape[1]] else: row = intensity df.loc[dt] = row # Rare case: month name glued to the end of a data row ("... 9 Jul"). trailing = _trailing_month_token(line) if trailing: month_num = _match_month_name(trailing) if month_num is not None: dt = dt.replace(month=month_num, day=1) elif _is_month_label_line(line): month_num = _match_month_name(line.strip()) if month_num is None: month_num = min(dt.month + 1, 12) dt = dt.replace(month=month_num, day=1) df.attrs['wavelength'] = wavelength return df def open_kyyyy_dat(path, fill_coordinates=False): """ Open Kislovodsk kyyyy.dat file. The file contains the parameters of each group of sunspots: observation time, coordinates, area, number of spots in the group. Parameters ---------- path : str or pathlib.Path Path to file. fill_coordinates : bool, optional Fill in the missing coordinates with the nearest values. Until 2009, the coordinates of the group were indicated only when passing through the central meridian. The default is False. Returns ------- df : pandas.DataFrame A data frame with sunspot groups data. """ sep = [0, 12, 19, 28, 36, 42, 49, 56, 63, 68] data = [] with open(path) as file: for line in file.readlines(): row = np.full(len(sep) - 1, np.nan) for i in range(row.shape[0]): try: row[i] = float(line[sep[i]: sep[i + 1]]) except ValueError: pass data.append(row) df = pd.DataFrame(data=data, columns=['date_time', 'group_number', 'lat', 'lon', 'r/R', 'area_uncorrected', 'area', 'area_max', 'n_spots']) dt = df['date_time'].astype(str).str.split('.') df['date_time'] = pd.to_datetime(dt.str[0]) + \ pd.to_timedelta(3600 * 24 * dt.str[1].astype(float) / 100, 's') df['date_time'] = df['date_time'].dt.tz_localize('UTC') for key in ['area_uncorrected', 'area', 'area_max', 'n_spots']: df[key].replace(np.nan, 0, inplace=True) if fill_coordinates: indices = df.index[(~df['r/R'].isna()) & df['lat'].isna()] for index in indices: dt = df.loc[index, 'date_time'] group = df.loc[index, 'group_number'] delta = (df['date_time'] - dt).abs().values mask = (df['group_number'] == group) & ~df['lat'].isna() index_closest = np.argmin(np.ma.masked_array(delta, ~mask)) df.loc[index, ['lat', 'lon']] = df.loc[index_closest, ['lat', 'lon']] return df