
import sys
from datetime import datetime
from rich.console import Console
console = Console(highlight=False)#,force_jupyter=False,width=999)

from cdasws.cdasws import CdasWs, TimeInterval
from cdasws.datarepresentation import DataRepresentation as dr
from cdasws.doi import get_doi_landing_page_url, get_doi_badge_url, get_doi_hdp_url

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

from IPython.core.display import HTML

today = datetime.now().strftime('%Y-%B-%d')

def display_link(
    url: str,
    text: str) -> None:
    display(HTML(f'<a href="{url}" target="_blank">{text}</a>'))

def display_image_link(
    url: str,
    image_url: str) -> None:
    display(HTML(f'<a href="{url}" target="_blank"><img src={image_url}/></a>'))
    

# Injects CSS to target the notebook's standard output stream container
display(HTML('''
<style>
div.output_area pre {
    font-size: 16px !important;
}
</style>
'''))


cdas = CdasWs()


%%time
# Fetch all observatory groups
obs_groups = cdas.get_observatory_groups()

# Print the names
tot_items=len(obs_groups)
#print(tot_items)
#print("Observatory -> IDs\n")
for indx, group in enumerate(obs_groups):
    #print("'%s'" %(group['Name']),',', end="")
    #print("\033[1;32m%s\033[0m -> %s" %(group['Name'], group['ObservatoryId']))
    if indx< tot_items-1: print("'\033[1;32m%s\033[0m'"  %(group['Name']),',',end="")# group['ObservatoryId']))
    if indx==tot_items-1: print("'\033[1;32m%s\033[0m'"  %(group['Name']))# group['ObservatoryId']))
        
    #print(f"\033[1;32m{group['Name']:<0}\033[0m -> {group['ObservatoryId']}")
    #print(f'{group["Name"]}')#"'%s'" %(group['Name']),',', end="")
    
print('='*100)

'ACE' ,'AMPTE' ,'ARTEMIS' ,'AWE' ,'Alouette' ,'Apollo' ,'Arase (ERG)' ,'Balloons' ,'BioSentinel' ,'CNOFS' ,'CRRES' ,'Cassini' ,'Cluster' ,'DMSP' ,'DSCOVR' ,'Demonstration and Science Experiments (DSX)' ,'Dynamics Explorer' ,'Equator-S' ,'FAST' ,'Formosat' ,'GOES' ,'GOLD' ,'GPS' ,'Genesis' ,'Geosynchronous Investigations' ,'Geotail' ,'Ground-Based Investigations' ,'Hawkeye' ,'Helio Ephemeris' ,'Helios' ,'IBEX' ,'ICON' ,'IMAGE' ,'IMAP' ,'IMP (All)' ,'ISEE' ,'ISIS' ,'ISS' ,'Interball' ,'LANL' ,'MAVEN' ,'MESSENGER' ,'MMS' ,'Mariner' ,'Mars Global Surveyor (MGS)' ,'Mars Science Laboratory (MSL)' ,'Moon' ,'NOAA' ,'New Horizons' ,'OMNI (Combined 1AU IP Data; Magnetic and Solar Indices)' ,'POES/MetOp' ,'Parker Solar Probe (PSP)' ,'Pioneer' ,'Polar' ,'REACH' ,'SAMPEX' ,'SOHO' ,'SOLAR' ,'ST5' ,'STEREO' ,'Smallsats/Cubesats' ,'Solar Orbiter' ,'Sounding Rockets' ,'THEMIS' ,'TIMED' ,'TRACERS' ,'TSS-1R' ,'TWINS' ,'Ulysses' ,'Van Allen Probes (RBSP)' ,'Voyager' ,'Wind'
====================================================================================================
CPU times: user 47.5 ms, sys: 16.3 ms, total: 63.8 ms
Wall time: 2.38 s


myobs=['ACE','Wind']
print('obs','->','[IDs]')
for indx, group in enumerate(obs_groups):
    if group['Name'] in myobs:
        print(group['Name'],'->',group['ObservatoryId'])

obs -> [IDs]
ACE -> ['AC', 'ACE', 'OMNI (1AU IP Data)']
Wind -> ['OMNI (1AU IP Data)', 'WI', 'WIND', 'Wi', 'Wind', 'wi']


%%time
# Fetch all instrument types
instrument_types = cdas.get_instrument_types()

# print the names
for group in instrument_types:
    print("'%s'" %(group['Name']))
    
print('='*100)

' Particles (space)'
'Activity Indices'
'Dust and Debris'
'Electric Fields (space)'
'Engineering'
'Ephemeris/Attitude/Ancillary'
'Ground-Based HF-Radars'
'Ground-Based Imagers'
'Ground-Based Magnetometers, Riometers, Sounders'
'Ground-Based VLF/ELF/ULF, Photometers'
'Housekeeping'
'Imagers (space)'
'Imaging and Remote Sensing (ITM/Earth)'
'Imaging and Remote Sensing (Magnetosphere/Earth)'
'Imaging and Remote Sensing (Sun)'
'Magnetic Fields (Balloon)'
'Magnetic Fields (space)'
'Particle Flux (space)'
'Particles (space)'
'Plasma and Solar Wind'
'Radio and Plasma Waves (space)'
'Space Weather Monitor'
====================================================================================================
CPU times: user 28.6 ms, sys: 6.15 ms, total: 34.8 ms
Wall time: 2.12 s


### user-specified ###
my_obs_inst={} # -> my_obs_inst['ID']=['instrument1','instrument2',...]
my_obs_inst['ACE']=['Plasma and Solar Wind', 'Magnetic Fields (space)']
my_obs_inst['Wind']=['Magnetic Fields (space)']

#myobservatory=['ACE', 'Wind']
#myinstrument =['Plasma and Solar Wind', 'Magnetic Fields (space)']


%%time
all_datasets = []
for obs in my_obs_inst:
    for inst in my_obs_inst[obs]:
        print('Observatory: %s' %(obs))
        print('Instrument: %s' %(inst))
        print('Dataset with the link -> Label')
        datasets = cdas.get_datasets(observatoryGroup=obs,instrumentType=inst)#myinstrument)
        if datasets:
            all_datasets.extend(datasets)
            
        lines=[]
        for index, dataset in enumerate(datasets):
            dataset_id = dataset["Id"]
            dataset_label = dataset["Label"]
            dataset_note = dataset['Notes']
            #dataset_note = dataset['DatasetLink']['Url']
            #skt_url = f"https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0SKELTABLES/{dataset_id}_00000000_v01.skt"
            #response = requests.get(skt_url)
            #skeleton_table_text = response.text
    
            #doi_info = dataset["Doi"]
            #
            #if doi_info:
            #    # Generate the landing page metadata URL
            #    metadata_url = get_doi_landing_page_url(doi_info)
            #else:
            #    # Fallback if no DOI info is provided directly in the dataset object
            #    metadata_url = f"https://nasa.gov{dataset_id}"
    
            #print('\033[1;32m%s\033[0m -> %s, %s' %(dataset_id, dataset_label,dataset_note))#, display_link(metadata_url, 'metadata')))
            #print("'\033[1m%s\033[0m' -> %s" %(dataset_id, dataset_label))#, display_link(metadata_url, 'metadata')))
            #print("%s" %(dataset_note))#, display_link(metadata_url, 'metadata')))
            #print('')
            #display(Markdown('\033[1;32m%s\033[0m -> %s' %(dataset_id, dataset_label)))#, display_link(metadata_url, 'metadata')))
            #print(f'{index}. {dataset_id}, {dataset_label}')
            lines.append("[bold green][link=%s]'%s'[/link][/] -> %s" %(dataset_note,dataset_id,dataset_label))
        console.print("\n".join(lines),justify='left', soft_wrap=True)
    
        print('='*100)
        print(' ')

Observatory: ACE
Instrument: Plasma and Solar Wind
Dataset with the link -> Label

'AC_AT_DEF' -> ACE Hourly RTN, GSE and J2000 GCI Attitude direction cosines - E. C. Stone (California Institute of Technology)                                                                                                                  
'AC_H0_SWE' -> ACE/SWEPAM Solar Wind Experiment 64-Second Level 2 Data - D. J. McComas (SWRI)                                                                                                                                                   
'AC_H2_SWE' -> ACE/SWEPAM Solar Wind Experiment 1-Hour Level 2 Data - D. J. McComas (SWRI)                                                                                                                                                      
'AC_H2_SWI' -> ACE/SWICS 1.1 Solar Wind 1-Hour Level 2 Data - G. Gloeckler (University of Maryland)                                                                                                                                             
'AC_H3_SW2' -> ACE/SWICS 2.0 Solar Wind 2-Hour Level 2 Data - G. Gloeckler (University of Maryland)                                                                                                                                             
'AC_H3_SWI' -> ACE/SWICS 1.1 Solar Wind 2-Hour Level 2 Data - G. Gloeckler (University of Maryland)                                                                                                                                             
'AC_H4_SWI' -> ACE/SWICS 1.1 Solar Wind 1-Day Level 2 Data - G. Gloeckler (University of Maryland)                                                                                                                                              
'AC_H5_SWI' -> ACE/SWICS 1.1 Solar Wind 2-Hour Level 2 Q-state distributions - G. Gloeckler (University of Maryland)                                                                                                                            
'AC_H6_SWI' -> ACE/SWICS Solar Wind Protons 12-min Level 2 Data - G. Gloeckler (University of Maryland)                                                                                                                                         
'AC_K0_GIFWALK' -> Links to ACE KP pre-generated survey and other plots - Polar-Wind-Geotail Ground System (NASA GSFC)                                                                                                                          
'AC_K0_SWE' -> K0 - ACE Solar Wind Experiment 5-Minute Key Parameters [PRELIM] - D. J. McComas (Southwest Research Institute)                                                                                                                   
'AC_K1_SWE' -> K1 - ACE Solar Wind Experiment 1-Hour Key Parameters [PRELIM] - D. J. McComas (Southwest Research Institute)                                                                                                                     
'AC_OR_DEF' -> ACE Daily GSE and J2000 GCI Position Data - E. C. Stone (California Institute of Technology)                                                                                                                                     
'OMNI2_H0_MRG1HR' -> OMNI Combined, Definitive, Hourly IMF and Plasma Data, and Energetic Proton Fluxes, Time-Shifted to the Nose of the Earth's Bow Shock, plus Solar and Magnetic Indices - J.H. King, N. Papitashvili (ADNET, NASA GSFC)     
'OMNI_COHO1HR_MERGED_MAG_PLASMA' -> OMNI Combined merged hourly magnetic field, plasma and ephermis data - J.H. King, N. Papatashvilli (AdnetSystems, NASA GSFC)                                                                                
'OMNI_HRO2_1MIN' -> OMNI Combined, Definitive 1-minute IMF and Definitive Plasma Data Time-Shifted to the Nose of the Earth's Bow Shock, plus Magnetic Indices - J.H. King, N. Papatashvilli (AdnetSystems, NASA GSFC)                          
'OMNI_HRO2_5MIN' -> OMNI Combined, Definitive 5-minute IMF and Definitive Plasma, and Energetic Proton Fluxes, Time-Shifted to the Nose  of the Earth's Bow Shock, plus Magnetic Indices - J.H. King, N. Papatashvilli (AdnetSystems, NASA GSFC)
'OMNI_HRO_1MIN' -> OMNI Combined, Definitive, 1-minute IMF and Plasma Data Time-Shifted to the Nose of the Earth's Bow Shock, plus Magnetic Indices - J.H. King, N. Papatashvilli (AdnetSystems, NASA GSFC)                                     
'OMNI_HRO_5MIN' -> OMNI Combined, Definitive, 5-minute IMF and Plasma, and Energetic Proton Fluxes, Time-Shifted to the Nose  of the Earth's Bow Shock, plus Magnetic Indices - J.H. King, N. Papatashvilli (AdnetSystems, NASA GSFC)

====================================================================================================
 
Observatory: ACE
Instrument: Magnetic Fields (space)
Dataset with the link -> Label

'AC_AT_DEF' -> ACE Hourly RTN, GSE and J2000 GCI Attitude direction cosines - E. C. Stone (California Institute of Technology)                                                                                                                  
'AC_H0_MFI' -> H0 - ACE Magnetic Field 16-Second Level 2 Data - N. Ness (Bartol Research Institute)                                                                                                                                             
'AC_H1_MFI' -> H1 - ACE Magnetic Field 4-Minute Level 2 Data - N. Ness (Bartol Research Institute)                                                                                                                                              
'AC_H2_MFI' -> H2 - ACE Magnetic Field 1-Hour Level 2 Data - N. Ness (Bartol Research Institute)                                                                                                                                                
'AC_H3_MFI' -> H3 - ACE Magnetic Field 1-Second Level 2 Data - N. Ness (Bartol Research Institute)                                                                                                                                              
'AC_K0_GIFWALK' -> Links to ACE KP pre-generated survey and other plots - Polar-Wind-Geotail Ground System (NASA GSFC)                                                                                                                          
'AC_K0_MFI' -> ACE Magnetic Field 5-Minute Key Parameters [PRELIM] - N. Ness (Bartol Research Institute)                                                                                                                                        
'AC_K1_MFI' -> ACE Magnetic Field 16-Second Key Parameters [PRELIM] - N. Ness (Bartol Research Institute)                                                                                                                                       
'AC_K2_MFI' -> K2 - ACE Magnetic Field 1-Hour Key Parameters [PRELIM] - N. Ness (Bartol Research Institute)                                                                                                                                     
'AC_OR_DEF' -> ACE Daily GSE and J2000 GCI Position Data - E. C. Stone (California Institute of Technology)                                                                                                                                     
'OMNI2_H0_MRG1HR' -> OMNI Combined, Definitive, Hourly IMF and Plasma Data, and Energetic Proton Fluxes, Time-Shifted to the Nose of the Earth's Bow Shock, plus Solar and Magnetic Indices - J.H. King, N. Papitashvili (ADNET, NASA GSFC)     
'OMNI_COHO1HR_MERGED_MAG_PLASMA' -> OMNI Combined merged hourly magnetic field, plasma and ephermis data - J.H. King, N. Papatashvilli (AdnetSystems, NASA GSFC)                                                                                
'OMNI_HRO2_1MIN' -> OMNI Combined, Definitive 1-minute IMF and Definitive Plasma Data Time-Shifted to the Nose of the Earth's Bow Shock, plus Magnetic Indices - J.H. King, N. Papatashvilli (AdnetSystems, NASA GSFC)                          
'OMNI_HRO2_5MIN' -> OMNI Combined, Definitive 5-minute IMF and Definitive Plasma, and Energetic Proton Fluxes, Time-Shifted to the Nose  of the Earth's Bow Shock, plus Magnetic Indices - J.H. King, N. Papatashvilli (AdnetSystems, NASA GSFC)
'OMNI_HRO_1MIN' -> OMNI Combined, Definitive, 1-minute IMF and Plasma Data Time-Shifted to the Nose of the Earth's Bow Shock, plus Magnetic Indices - J.H. King, N. Papatashvilli (AdnetSystems, NASA GSFC)                                     
'OMNI_HRO_5MIN' -> OMNI Combined, Definitive, 5-minute IMF and Plasma, and Energetic Proton Fluxes, Time-Shifted to the Nose  of the Earth's Bow Shock, plus Magnetic Indices - J.H. King, N. Papatashvilli (AdnetSystems, NASA GSFC)

====================================================================================================
 
Observatory: Wind
Instrument: Magnetic Fields (space)
Dataset with the link -> Label

'OMNI2_H0_MRG1HR' -> OMNI Combined, Definitive, Hourly IMF and Plasma Data, and Energetic Proton Fluxes, Time-Shifted to the Nose of the Earth's Bow Shock, plus Solar and Magnetic Indices - J.H. King, N. Papitashvili (ADNET, NASA GSFC)     
'OMNI_COHO1HR_MERGED_MAG_PLASMA' -> OMNI Combined merged hourly magnetic field, plasma and ephermis data - J.H. King, N. Papatashvilli (AdnetSystems, NASA GSFC)                                                                                
'OMNI_HRO2_1MIN' -> OMNI Combined, Definitive 1-minute IMF and Definitive Plasma Data Time-Shifted to the Nose of the Earth's Bow Shock, plus Magnetic Indices - J.H. King, N. Papatashvilli (AdnetSystems, NASA GSFC)                          
'OMNI_HRO2_5MIN' -> OMNI Combined, Definitive 5-minute IMF and Definitive Plasma, and Energetic Proton Fluxes, Time-Shifted to the Nose  of the Earth's Bow Shock, plus Magnetic Indices - J.H. King, N. Papatashvilli (AdnetSystems, NASA GSFC)
'OMNI_HRO_1MIN' -> OMNI Combined, Definitive, 1-minute IMF and Plasma Data Time-Shifted to the Nose of the Earth's Bow Shock, plus Magnetic Indices - J.H. King, N. Papatashvilli (AdnetSystems, NASA GSFC)                                     
'OMNI_HRO_5MIN' -> OMNI Combined, Definitive, 5-minute IMF and Plasma, and Energetic Proton Fluxes, Time-Shifted to the Nose  of the Earth's Bow Shock, plus Magnetic Indices - J.H. King, N. Papatashvilli (AdnetSystems, NASA GSFC)           
'WI_H0_MFI' -> Wind Magnetic Fields Investigation: 3 sec, 1 min, and hourly Definitive Data. - A. Koval (UMBC, NASA/GSFC)                                                                                                                       
'WI_H1_SWE' -> Wind Solar Wind Experiment, 92-sec Solar Wind Alpha and Proton Anisotropy Analysis - K. Ogilvie (NASA GSFC)                                                                                                                      
'WI_H1_SWE_RTN' -> Solar wind proton and alpha parameters, including anisotropic temperatures, derived by non-linear fitting of the measurements and with moment techniques. Data in RTN - Bennett Maruca (University of Delaware)              
'WI_H2_MFI' -> Wind Magnetic Fields Investigation, High-resolution Definitive Data - A. Koval (UMBC, NASA/GSFC)                                                                                                                                 
'WI_H3-RTN_MFI' -> Wind Magnetic Fields Investigation: 3 sec, 1 min, and hourly Definitive Data (RTN). - A. Koval (UMBC, NASA/GSFC)                                                                                                             
'WI_H4-RTN_MFI' -> Wind Magnetic Fields Investigation, High-resolution Definitive Data (RTN) - A. Koval (UMBC, NASA/GSFC)                                                                                                                       
'WI_K0_GIFWALK' -> Links to Wind KP pre-generated survey and other plots - Polar-Wind-Geotail Ground System (NASA GSFC)                                                                                                                         
'WI_K0_MFI' -> Wind Magnetic Fields Investigation, Key Parameters [PRELIM] - R. Lepping (NASA/GSFC)                                                                                                                                             
'WI_K1-RTN_MFI' -> Wind Magnetic Fields Investigation, Key Parameters in RTN [PRELIM] - Andriy Koval (UMBC&GSFC)

====================================================================================================
 
CPU times: user 96.3 ms, sys: 15.6 ms, total: 112 ms
Wall time: 6.15 s


### user-specified ###
mydataset=['WI_H0_MFI','AC_H0_MFI','AC_H0_SWE']


%%time
mydois=[]
for mydset in mydataset:
    for index, dataset in enumerate(all_datasets):
        dataset_id = dataset["Id"]
        dataset_label = dataset["Label"]
    
        if mydset==dataset_id:
        #if dataset_id in mydataset:  # select this dataset for use below
            doi=dataset['Doi']
            #print(doi)
            mydois.append(dataset['Doi'])
            print(f'\nSelected dataset: {dataset_id} (doi:{doi})')
            print(f"Available Period: {dataset['TimeInterval']}")
            display_image_link(get_doi_landing_page_url(doi), get_doi_badge_url(doi))
            display_link(get_doi_hdp_url(doi), 'HDP Description')
            display_link(dataset['Notes'], 'CDAWeb Notes')
            display(HTML(f'<b>Data Citation</b>: {cdas.get_citation(doi)}.  Accessed on {today}.'))
            
            #variables = cdas.get_variables(doi)
            #var_names = []
            #for index, variable in enumerate(variables):
            #    name = variable['Name']
            #    var_names.append(name)
            #    print("'%s'" %(variable['Name']),"->", variable['LongDescription'])
            
            print('='*100)
            print(' ')

Selected dataset: WI_H0_MFI (doi:10.48322/av38-wn55)
Available Period: {'Start': '1994-11-13T00:00:30.000Z', 'End': '2026-09-12T23:59:30.000Z'}

====================================================================================================
 

Selected dataset: AC_H0_MFI (doi:10.48322/e0dc-0h53)
Available Period: {'Start': '1997-09-02T00:00:12.000Z', 'End': '2026-08-04T23:59:59.000Z'}

====================================================================================================
 

Selected dataset: AC_H0_SWE (doi:10.48322/tsfn-6v47)
Available Period: {'Start': '1998-02-04T00:00:31.000Z', 'End': '2024-07-09T23:59:13.000Z'}

====================================================================================================
 
CPU times: user 193 ms, sys: 47.6 ms, total: 241 ms
Wall time: 4.57 s


%%time
for indx,dset in enumerate(mydataset):
    variables = cdas.get_variables(dset)
    print('Dataset: %s' %(mydataset[indx]))
    
    search_window = TimeInterval("1900-01-01T00:00:00Z", "2027-01-01T00:00:00Z")
    filtered_inventory = cdas.get_inventory(mydataset[indx], timeInterval=search_window)

    for interval in filtered_inventory:
        print(f"Available from: \033[1;32m{interval.start}\033[0m to \033[1;32m{interval.end}\033[0m")
    print(' ')
    var_names = []
    for index, variable in enumerate(variables):
        name = variable['Name']
        var_names.append(name)
        print("\033[1;32m'%s'\033[0m" %(variable['Name']),"->", variable['LongDescription'])
    #    print(f'{index}. {name}, {variable["LongDescription"]}')
    print('='*100)
    print(' ')

Dataset: WI_H0_MFI
Available from: 1994-11-13 00:00:30+00:00 to 2006-11-23 23:59:30+00:00
Available from: 2006-11-28 00:00:30+00:00 to 2009-02-20 23:59:30+00:00
Available from: 2009-02-24 00:00:30+00:00 to 2014-10-27 23:59:30+00:00
Available from: 2014-11-21 00:00:30+00:00 to 2023-07-14 23:59:30+00:00
Available from: 2023-07-26 00:00:30+00:00 to 2026-07-24 23:59:30+00:00
Available from: 2026-07-26 00:00:30+00:00 to 2026-09-12 23:59:30+00:00
 
'BF1' -> Magnetic field magnitude (1 min)
'BF1LOG' -> Magnetic field magnitude (1 min - log scaled)
'BRMSF1' -> RMS magnitude (1 min)
'BGSM' -> Magnetic field vector in GSM cartesian coordinates (1 min)
'BRMSGSM' -> RMS vector in GSM coordinates (1 min)
'BGSE' -> Magnetic field vector in GSE cartesian coordinates (1 min)
'BGSEa' -> Magnetic field vector in GSE angular coordinates (1 min)
'BRMSGSE' -> RMS vector in GSE coordinates (1 min)
'DIST' -> Distance from the center of the earth (Define Re = 6378km) (1 min)
'PGSM' -> Position vector in GSM coordinates (Define Re = 6378km) (1 min)
'PGSE' -> Position vector in GSE coordinates (Define Re = 6378km) (1 min)
'B3F1' -> Magnetic field magnitude (3 sec)
'B3F1LOG' -> Magnetic field magnitude (3 sec - log scaled)
'B3RMSF1' -> RMS magnitude (3 sec)
'B3GSM' -> Magnetic field vector in GSM cartesian coordinates (3 sec)
'B3RMSGSM' -> RMS vector in GSM coordinates (3 sec)
'B3GSE' -> Magnetic field vector in GSE cartesian coordinates (3 sec)
'B3GSEa' -> Magnetic field vector in GSE angular coordinates (3 sec)
'B3RMSGSE' -> RMS vector in GSE coordinates (3 sec)
'B1F1' -> Magnetic field magnitude (1 hour)
'B1F1LOG' -> Magnetic field magnitude (1 hour - log scaled)
'B1RMSF1' -> RMS magnitude (1 hour)
'B1GSM' -> Magnetic field vector in GSM cartesian coordinates (1 hour)
'B1RMSGSM' -> RMS vector in GSM coordinates (1 hour)
'B1GSE' -> Magnetic field vector in GSE cartesian coordinates (1 hour)
'B1GSEa' -> Magnetic field vector in GSE angular coordinates (1 hour)
'B1RMSGSE' -> RMS vector in GSE coordinates (1 hour)
'DIST1' -> Distance from the center of the earth (Define Re = 6378km) (1 hour)
'P1GSM' -> Position vector in GSM coordinates (Define Re = 6378km) (1 hour)
'P1GSE' -> Position vector in GSE coordinates (Define Re = 6378km) (1 hour)
'DISTV' -> Time axis label: Distance from the center of the earth (Define Re = 6378km) (1 min)
'PGSMV' -> Time axis label: Position vector in GSM coordinates (Define Re = 6378km) (1 min)
'PGSEV' -> Time axis label: Position vector in GSE coordinates (Define Re = 6378km) (1 min)
'DIST1V' -> Time axis label: Distance from the center of the earth (Define Re = 6378km) (1 hour)
'P1GSMV' -> Time axis label: Position vector in GSM coordinates (Define Re = 6378km) (1 hour)
'P1GSEV' -> Time axis label: Position vector in GSE coordinates (Define Re = 6378km) (1 hour)
====================================================================================================
 
Dataset: AC_H0_MFI
Available from: 1997-09-02 00:00:12+00:00 to 2024-10-01 23:59:54+00:00
Available from: 2024-10-02 23:59:59+00:00 to 2026-08-04 23:59:59+00:00
 
'Magnitude' -> B-field magnitude
'BGSEc' -> Magnetic Field Vector in GSE Cartesian coordinates (16 sec)
'BGSM' -> Magnetic field vector in GSM coordinates (16 sec)
'dBrms' -> RMS of Magnetic Field (16 sec period)
'SC_pos_GSE' -> ACE s/c position, 3 comp. in GSE coord.
'SC_pos_GSM' -> ACE s/c position, 3 comp. in GSM coord.
====================================================================================================
 
Dataset: AC_H0_SWE
Available from: 1998-02-04 00:00:31+00:00 to 2024-04-19 23:59:27+00:00
Available from: 2024-04-21 00:00:31+00:00 to 2024-07-09 23:59:13+00:00
 
'Np' -> Solar Wind Proton Number Density, scalar
'Vp' -> Solar Wind Bulk Speed
'Tpr' -> radial component of the proton temperature
'alpha_ratio' -> alpha to proton density ratio
'V_GSE' -> Solar Wind Velocity in GSE coord., 3 components
'V_RTN' -> Solar Wind Velocity in RTN coord., 3 components
'V_GSM' -> Solar Wind Velocity in GSM coord., 3 comp.
'SC_pos_GSE' -> ACE s/c position, 3 comp. in GSE coord.
'SC_pos_GSM' -> ACE s/c position, 3 comp. in GSM coord.
====================================================================================================
 
CPU times: user 85.9 ms, sys: 29.1 ms, total: 115 ms
Wall time: 6.33 s


#show again the name of mydatasets for a reference 
print(mydataset)

['WI_H0_MFI', 'AC_H0_MFI', 'AC_H0_SWE']


%%time
### user-specified ###
observatory_config={
    'ACE': {
        'time_periods':{
            "evnt1": ('1998-02-04T00:00:00Z', '1999-02-04T00:00:00Z'),
        },
        'datasets':{
            'AC_H0_MFI': ['BGSM'],
            'AC_H0_SWE': ['Np', 'Vp', 'V_GSM']
        },
    },
    'Wind': {
        'time_periods':{
            "evnt1": ('1998-02-04T00:00:00Z', '1999-02-04T00:00:00Z'),
            "evnt2": ('2001-02-04T00:00:00Z', '2002-02-04T00:00:00Z')
        },
        'datasets':{
            'WI_H0_MFI': ['BGSM']
        },
    }
}

####!!! when the time periods are the same, !!!###
#time_periods={
#    "evnt1": ('1998-02-04T00:00:00Z', '1999-02-04T00:00:00Z'),
#    "evnt2": ('2001-02-04T00:00:00Z', '2002-02-04T00:00:00Z')
#}
#observatory_config={
#    'ACE': {
#        'time_periods':time_periods,
#        'datasets':{
#            'AC_H0_MFI': ['BGSM'],
#            'AC_H0_SWE': ['Np', 'Vp', 'V_GSM']
#        },
#    },
#    'Wind': {
#        'time_periods':time_periods,
#        'datasets':{
#            'WI_H0_MFI': ['BGSM']
#        },
#    }
#}
####!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!###
#######################

## request the data representation as xarray
obs_dat={}#{sc: {} for sc in observatory_config}
for sc_name,config in observatory_config.items():
    obs_dat[sc_name]={}
    for evnt, (strt,end) in config['time_periods'].items():
        obs_dat[sc_name][evnt]={}
        #print(f"Period: {evnt} -> Start: {strt}, End: {end}")
        for dset,variables in config['datasets'].items():
            #print(evnt,dset,variables)
            status, result = cdas.get_data(dset, variables, strt, end, dataRepresentation = dr.XARRAY)
            keys=list(result.indexes.keys())
            result_rename=result.rename({keys[0]: "time"})
            print('Event:',evnt,'-','Observatory:',sc_name,'-','Dataset:', dset)
            if status['http']['status_code'] == 200:
                obs_dat[sc_name][evnt][dset]=result_rename
                print('Donwload Success!')
            else:
                print(f"Error! HTTP Status Code: {status['http']['status_code']}")
            print('='*100)
            print('')

Event: evnt1 - Observatory: ACE - Dataset: AC_H0_MFI
Donwload Success!
====================================================================================================

Event: evnt1 - Observatory: ACE - Dataset: AC_H0_SWE
Donwload Success!
====================================================================================================

Event: evnt1 - Observatory: Wind - Dataset: WI_H0_MFI
Donwload Success!
====================================================================================================

Event: evnt2 - Observatory: Wind - Dataset: WI_H0_MFI
Donwload Success!
====================================================================================================

CPU times: user 51.1 s, sys: 2.87 s, total: 54 s
Wall time: 1min 28s


plt.figure(figsize=(20,5))
plt.plot(obs_dat['ACE']['evnt1']['AC_H0_SWE']['time'],obs_dat['ACE']['evnt1']['AC_H0_SWE']['Np'])
plt.ylabel('proton density [cm$^{-3}$]')
plt.xlim('1998-02-04T00:00:00Z','1998-02-10T00:00:00Z')
plt.ylim(0,50)
plt.show()

plt.figure(figsize=(20,5))
plt.plot(obs_dat['ACE']['evnt1']['AC_H0_SWE']['time'],obs_dat['ACE']['evnt1']['AC_H0_SWE']['V_GSM'][:,0])
plt.plot(obs_dat['ACE']['evnt1']['AC_H0_SWE']['time'],obs_dat['ACE']['evnt1']['AC_H0_SWE']['V_GSM'][:,1])
plt.plot(obs_dat['ACE']['evnt1']['AC_H0_SWE']['time'],obs_dat['ACE']['evnt1']['AC_H0_SWE']['V_GSM'][:,2])
plt.ylabel('magnetic fields [nT]')
plt.xlim('1998-02-04T00:00:00Z','1998-02-10T00:00:00Z')
plt.ylim(-700,100)
plt.show()

plt.figure(figsize=(20,5))
plt.plot(obs_dat['ACE']['evnt1']['AC_H0_MFI']['time'],obs_dat['ACE']['evnt1']['AC_H0_MFI']['BGSM'][:,0])
plt.plot(obs_dat['ACE']['evnt1']['AC_H0_MFI']['time'],obs_dat['ACE']['evnt1']['AC_H0_MFI']['BGSM'][:,1])
plt.plot(obs_dat['ACE']['evnt1']['AC_H0_MFI']['time'],obs_dat['ACE']['evnt1']['AC_H0_MFI']['BGSM'][:,2])
plt.ylabel('magnetic fields [nT]')
plt.xlim('1998-02-04T00:00:00Z','1998-02-10T00:00:00Z')
plt.ylim(-20,20)
plt.show()

plt.figure(figsize=(20,5))
plt.plot(obs_dat['Wind']['evnt1']['WI_H0_MFI']['time'],obs_dat['Wind']['evnt1']['WI_H0_MFI']['BGSM'][:,0])
plt.plot(obs_dat['Wind']['evnt1']['WI_H0_MFI']['time'],obs_dat['Wind']['evnt1']['WI_H0_MFI']['BGSM'][:,1])
plt.plot(obs_dat['Wind']['evnt1']['WI_H0_MFI']['time'],obs_dat['Wind']['evnt1']['WI_H0_MFI']['BGSM'][:,2])
plt.ylabel('magnetic fields [nT]')
plt.xlim('1998-02-04T00:00:00Z','1998-02-10T00:00:00Z')
plt.ylim(-20,20)
plt.show()

plt.figure(figsize=(20,5))
plt.plot(obs_dat['Wind']['evnt2']['WI_H0_MFI']['time'],obs_dat['Wind']['evnt2']['WI_H0_MFI']['BGSM'][:,0])
plt.plot(obs_dat['Wind']['evnt2']['WI_H0_MFI']['time'],obs_dat['Wind']['evnt2']['WI_H0_MFI']['BGSM'][:,1])
plt.plot(obs_dat['Wind']['evnt2']['WI_H0_MFI']['time'],obs_dat['Wind']['evnt2']['WI_H0_MFI']['BGSM'][:,2])
plt.ylabel('magnetic fields [nT]')
plt.xlim('2001-02-04T00:00:00Z','2001-02-10T00:00:00Z')
plt.ylim(-20,20)
plt.show()


%%time
#https://angeo.copernicus.org/articles/19/1545/2001/angeo-19-1545-2001.pdf
#may not be the same instrument
dataset = 'C1_CP_CIS-CODIF_HS_H1_PEF'
var_names = cdas.get_variable_names(dataset)
start_time, end_time = '2001-02-02T09:00:00Z', '2001-02-02T10:00:00Z'

status, dat_3d = cdas.get_data(dataset, var_names, start_time, end_time, dataRepresentation=dr.XARRAY)
print(status)
print(var_names)

{'http': {'status_code': 200}, 'cdas': {'status': [], 'message': [], 'warning': [], 'error': []}}
['duration__C1_CP_CIS_CODIF_HS_H1_PEF', 'cis_mode__C1_CP_CIS_CODIF_HS_H1_PEF', 'tm_product__C1_CP_CIS_CODIF_HS_H1_PEF', 'codif_mcp_hv__C1_CP_CIS_CODIF_HS_H1_PEF', 'codif_acc_hv__C1_CP_CIS_CODIF_HS_H1_PEF', 'ions_3d__C1_CP_CIS_CODIF_HS_H1_PEF']
CPU times: user 487 ms, sys: 91.8 ms, total: 579 ms
Wall time: 2.45 s


from IPython.display import display, clear_output
import time

print(dat_3d.keys())
th=dat_3d['theta__C1_CP_CIS_CODIF_HS_H1_PEF'].values
ph=dat_3d['phi__C1_CP_CIS_CODIF_HS_H1_PEF'].values
en=dat_3d['energy_table__C1_CP_CIS_CODIF_HS_H1_PEF'].values

#print(th.attrs)
#print(ph.attrs)

print(dat_3d['ions_3d__C1_CP_CIS_CODIF_HS_H1_PEF'].shape)

V, Phi = np.meshgrid(np.sqrt(en), np.pi*ph/180)

fig, ax = plt.subplots(figsize=(10,10),subplot_kw={'projection': 'polar'})

it=10
for indx in range(len(th)):
    ax.cla()
    contour = ax.contourf(Phi, V, np.log10(dat_3d['ions_3d__C1_CP_CIS_CODIF_HS_H1_PEF'][it,indx,:,:]+1e-10), levels=100, cmap='jet')
    clear_output(wait=True)
    display(fig)
    time.sleep(1)

clear_output(wait=True)

How to read data using the cdasws Python package¶

Workflow¶

Initialize the client¶

List the name of observatories¶

List the ID of chosen observatories¶

List the name of instrument types¶

List the dataset IDs¶

Show the URL/DOI of the selected dataset¶

List the variables of the selected dataset¶

Download data¶

3D Ion Vleocity Distribution Function¶