In [1]:
import glances
import numpy as np
import multiprocessing as mp
CORES=40
In [2]:
# https://docs.google.com/spreadsheets/d/1rg5P26ArIJVSojHipEJ4Wi7-mhgeA1bbFGqYoWAsVP4/edit#gid=0
# convdate %Y%m%d %Y/%j 20160117 20160119 20160218 20160224 20160318 20160324 20160411 20160414 20160417 20160419 20160505 20160506 20160509 20160510 20160511 20160512 20160513 20160517 20160615 20160913 20160920 20160921 20160922 20160929 20161024 20161004 20161210
anomstr = """
2016/017
2016/019
2016/049
2016/055
2016/078
2016/084
2016/102
2016/105
2016/108
2016/110
2016/126
2016/127
2016/130
2016/131
2016/132
2016/133
2016/134
2016/138
2016/167
2016/257
2016/264
2016/265
2016/266
2016/273
2016/298
2016/278
2016/345
"""
In [3]:
anoms = set(x.strip() for x in anomstr.split())
anoms
Out[3]:
{'2016/017',
 '2016/019',
 '2016/049',
 '2016/055',
 '2016/078',
 '2016/084',
 '2016/102',
 '2016/105',
 '2016/108',
 '2016/110',
 '2016/126',
 '2016/127',
 '2016/130',
 '2016/131',
 '2016/132',
 '2016/133',
 '2016/134',
 '2016/138',
 '2016/167',
 '2016/257',
 '2016/264',
 '2016/265',
 '2016/266',
 '2016/273',
 '2016/278',
 '2016/298',
 '2016/345'}
In [4]:
inputs = tuple([x.strip() for x in open('input.txt').readlines()])
# inputs = tuple(x for x in inputs if x not in anoms)  # ditch known anomaly days, optionally
inputs[:4]
Out[4]:
('2016/001', '2016/002', '2016/003', '2016/004')
In [5]:
import os
join = os.path.join
from glob import glob
import re
RE_gXXX = re.compile('g(\d\d\d).txt')
for dur in inputs:
    for fn in glob(join(dur,'gl*txt')):
        gxxx, = RE_gXXX.findall(fn)
        yyyy,jjj = dur.split('/')
        nn = join('all', '%sj%sg%s' % (yyyy,jjj,gxxx))
        try:
            if not os.path.exists(nn):
                os.symlink(join('..', fn), nn)
        except:
            print("%s ???" % fn)
In [6]:
def load_glances(dur):
    try:
        pfx = dur.replace('/','j')
        g = glances.dataframe_from_glances(os.path.join('all', pfx + '*'))
    except:
        g = None
        print("oops " + pfx)
    return (pfx, g)
In [7]:
import pandas as pd
pool = mp.Pool(CORES)
dfs = dict(pool.map(load_glances, inputs))
pool.close()
pool.join()

oops 2017j001
oops 2017j003
oops 2017j002
oops 2016j032
oops 2016j008
In [8]:
df = pd.concat(x for x in dfs.values() if x is not None)
In [9]:
df.shape
Out[9]:
(7200, 2200)
In [10]:
def sos(jxxx):
    rads = ['rad_lw', 'rad_mw', 'rad_sw']
    geos = ['lat', 'lon', 'sat_range']
    flgs = [] # 'geo_qual', '11b_qual']
    zult = {}
    for rad in rads:
        zult[rad] = (jxxx.ix[:][rad,'max_diff'] > 1e-4).nonzero()
    for geo in geos:
        zult[geo] = (jxxx.ix[:][geo,'max_diff'] > 0.000025).nonzero() # about 1km max
    for flg in flgs:
        zult[flg] = jxxx.ix[:][flg,'max_diff'].nonzero()
    return zult
In [11]:
q = sos(df)
q
Out[11]:
{'lat': (array([5563, 5564, 5565, 5566, 5567, 5568, 5569, 5570, 5571, 5572, 5608]),),
 'lon': (array([ 482,  809,  981, 1048, 1265, 1593, 2885, 3212, 4695, 4826, 4978,
         5046, 5130, 5524, 5563, 5564, 5565, 5566, 5567, 5568, 5569, 5570,
         5571, 5572, 5608, 5807, 5875, 6987, 7139]),),
 'rad_lw': (array([  43, 2413, 2414, 3876, 3877, 3880, 3881, 3891, 3892, 3893, 4825,
         4826, 4858, 4859, 5608, 5609]),),
 'rad_mw': (array([ 358,  359, 3876, 3877, 3880, 3881, 3891, 3892, 3893, 4825, 4826,
         4858, 4859, 5608, 5609]),),
 'rad_sw': (array([ 358,  359, 2361, 3876, 3877, 3891, 3892, 3893, 4825, 4826, 4858,
         4859, 5608, 5609]),),
 'sat_range': (array([   0,    1,    2, ..., 7197, 7198, 7199]),)}
In [12]:
df.ix['2016j004g119'][:,'max_diff']
Out[12]:
asc_flag                      0
for_num                       0
fov_num                       0
geo_qual                      0
instrument_state              0
l1b_qual                      0
land_frac                     0
lat                 1.90735e-06
lat_bnds             3.8147e-06
lat_geoid           1.90735e-06
lon                 1.52588e-05
lon_bnds            1.52588e-05
lon_geoid           1.52588e-05
nedn_lw                       0
nedn_mw                       0
nedn_sw                       0
obs_time_tai                  0
obs_time_utc                  0
rad_lw                        0
rad_mw               0.00790787
rad_sw              0.000557303
sat_alt                       0
sat_att             4.74683e-08
sat_azi             3.05176e-05
sat_pos                       0
sat_range                 0.125
sat_vel                       0
sat_zen              3.8147e-06
scan_mid_time                 0
scan_sweep_dir                0
sol_azi             1.52588e-05
sol_zen             1.52588e-05
subsat_lat                    0
subsat_lon                    0
sun_glint_dist              NaN
sun_glint_lat               NaN
sun_glint_lon               NaN
surf_alt                      0
surf_alt_sdev                 0
view_ang             3.8147e-06
Name: 2016j004g119, dtype: object
In [13]:
[max(df.ix[:][lms,'max_diff']) for lms in ('rad_lw', 'rad_mw', 'rad_sw')]
Out[13]:
[3232.9400000000001, 0.039579499999999997, 0.0022378900000000002]
In [14]:
# rough numerical epsilons for 6.5 digits
follo = [(df.ix[:][lms,'max_diff'] > v).nonzero() for (lms,v) in zip(('rad_lw', 'rad_mw', 'rad_sw'), (0.0001, 0.0001, 0.00001))]
follo
Out[14]:
[(array([  43, 2413, 2414, 3876, 3877, 3880, 3881, 3891, 3892, 3893, 4825,
         4826, 4858, 4859, 5608, 5609]),),
 (array([ 358,  359, 3876, 3877, 3880, 3881, 3891, 3892, 3893, 4825, 4826,
         4858, 4859, 5608, 5609]),),
 (array([ 358,  359, 2361, 3876, 3877, 3891, 3892, 3893, 4825, 4826, 4858,
         4859, 5608, 5609]),)]
In [15]:
wtf = [(df.ix[:][lms,'max_diff'] > v).nonzero() for (lms,v) in zip(('rad_lw', 'rad_mw', 'rad_sw'), (0.001, 0.001, 0.0001))]
wtf
Out[15]:
[(array([  43, 2413, 2414, 3876, 3877, 3880, 3881, 3891, 3892, 3893, 4825,
         4826, 4858, 4859, 5608, 5609]),),
 (array([ 358,  359, 3876, 3877, 3880, 3881, 3891, 3892, 3893, 4825, 4826,
         4858, 4859, 5608, 5609]),),
 (array([ 358,  359, 2361, 3876, 3877, 3891, 3892, 3893, 4825, 4826, 4858,
         4859, 5608, 5609]),)]
In [16]:
# check r-squared correlation
rsq  = [(df.ix[:][lms,'r-squared correlation'] < v).nonzero() for (lms,v) in zip(('rad_lw', 'rad_mw', 'rad_sw'), (0.9999, 0.9999, 0.9999))]
rsq
Out[16]:
[(array([43]),), (array([], dtype=int64),), (array([], dtype=int64),)]
In [17]:
df.rad_lw[df.rad_lw['max_diff'] > 0.0001].transpose()
Out[17]:
2016j005g044 2016j024g014 2016j024g015 2016j028g037 2016j028g038 2016j028g041 2016j028g042 2016j028g052 2016j028g053 2016j028g054 2016j010g026 2016j010g027 2016j010g059 2016j010g060 2016j017g089 2016j017g090
a_finite_count 8711550 8711550 8711550 3291030 8711550 8711550 8711550 8517960 2071413 8711550 8130780 8711550 8517960 8492148 1742310 8711550
a_finite_fraction 1 1 1 0.3777778 1 1 1 0.9777778 0.2377778 1 0.9333333 1 0.9777778 0.9748148 0.2 1
a_missing_count 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
a_missing_fraction 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
a_missing_value 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36
a_nan_count 0 0 0 5420520 0 0 0 193590 6640137 0 580770 0 193590 219402 6969240 0
a_nan_fraction 0 0 0 0.6222222 0 0 0 0.02222222 0.7622222 0 0.06666667 0 0.02222222 0.02518519 0.8 0
b_finite_count 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550 1742310 8711550
b_finite_fraction 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0.2 1
b_missing_count 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
b_missing_fraction 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
b_missing_value 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36
b_nan_count 0 0 0 0 0 0 0 0 0 0 0 0 0 0 6969240 0
b_nan_fraction 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.8 0
common_finite_count 8711550 8711550 8711550 3291030 8711550 8711550 8711550 8517960 2071413 8711550 8130780 8711550 8517960 8492148 1742310 8711550
common_finite_fraction 1 1 1 0.3777778 1 1 1 0.9777778 0.2377778 1 0.9333333 1 0.9777778 0.9748148 0.2 1
common_missing_count 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
common_missing_fraction 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
common_nan_count 0 0 0 0 0 0 0 0 0 0 0 0 0 0 6969240 0
common_nan_fraction 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.8 0
correlation 0.991963 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
diff_outside_epsilon_count 717 128615 182486 2516149 2709178 408106 214960 2709346 2071183 580544 2708880 2708884 2708645 2683074 1741437 967346
diff_outside_epsilon_fraction 8.230453e-05 0.01476373 0.02094759 0.7645476 0.3109869 0.04684654 0.02467529 0.3180745 0.999889 0.06664072 0.3331636 0.3109532 0.3179922 0.3159476 0.9994989 0.1110418
epsilon 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
epsilon_percent NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
finite_in_only_one_count 0 0 0 5420520 0 0 0 193590 6640137 0 580770 0 193590 219402 0 0
finite_in_only_one_fraction 0 0 0 0.6222222 0 0 0 0.02222222 0.7622222 0 0.06666667 0 0.02222222 0.02518519 0 0
max_a 105.667 124.291 98.9222 83.5734 103.868 134.126 135.081 119.413 81.661 98.9868 87.5305 100.407 106.978 95.4691 76.8047 88.6718
max_b 484.477 124.291 98.9222 88.4748 103.868 134.126 135.081 119.413 82.6339 98.9868 87.5305 100.407 106.978 95.4691 76.815 88.6718
max_delta 454.943 0.0278664 0.0301857 0.236595 0.237318 0.0227547 0.0240746 0.297855 0.271437 0.0910816 0.0741272 0.0827713 0.0680733 0.0640831 0.0713577 0.0459404
max_diff 3232.94 0.0278664 0.0301857 0.24143 0.273169 0.0326271 0.0330849 0.308235 0.298214 0.117607 0.0980377 0.0887527 0.0680733 0.0640831 0.0713577 0.0459404
mean_a 47.5832 63.0505 45.3816 37.3533 46.9861 83.6369 57.076 50.6066 30.9578 43.2353 40.6948 43.2834 45.8045 39.8572 34.578 39.3302
mean_b 47.5847 63.0505 45.3816 40.9314 46.9863 83.6369 57.076 50.5036 36.8193 43.2352 40.7499 43.2833 45.6442 39.8194 34.5776 39.3301
mean_delta 0.00154281 8.82095e-07 8.02439e-07 0.00263809 0.000237034 5.09241e-06 -7.42119e-06 5.69766e-05 -0.00513598 -8.843e-05 0.00010352 -5.44912e-05 0.000166402 -0.000201952 -0.00037236 -4.12139e-05
mean_diff 0.0110953 1.6091e-05 2.53539e-05 0.00512564 0.00189995 0.000110296 4.48655e-05 0.00188752 0.00859469 0.000203716 0.000883263 0.00084523 0.000735451 0.00072575 0.00245571 0.000185637
median_a 45.563 61.6027 46.0037 37.2562 45.2114 90.4129 51.5631 47.9276 30.2021 43.0159 41.3543 43.2082 45.3029 39.0656 34.8692 38.648
median_b 45.5656 61.6027 46.0037 40.7476 45.2114 90.4129 51.5632 47.8979 36.6807 43.0158 41.3726 43.2082 45.1455 39.0412 34.8688 38.6479
median_delta 0 0 0 0.000619888 0 0 0 0 -0.0044899 0 0 0 0 0 -0.000360489 0
median_diff 0 0 0 0.00269318 0 0 0 0 0.00612259 0 0 0 0 0 0.00177383 0
min_a 7.52624 15.1711 12.3553 7.64238 8.20328 11.3489 5.8639 11.9115 8.28849 7.74516 10.1363 8.29676 10.1369 8.66607 7.85827 7.48006
min_b -3189.67 15.1711 12.3553 7.59 8.20056 11.3489 5.8639 11.9115 8.29733 7.74516 9.51183 8.29676 9.91587 8.66614 7.85819 7.48006
min_delta -3232.94 -0.0222778 -0.0204544 -0.24143 -0.273169 -0.0326271 -0.0330849 -0.308235 -0.298214 -0.117607 -0.0980377 -0.0887527 -0.0565147 -0.0570679 -0.0680695 -0.0452042
mismatch_points_count 717 128615 182486 7936669 2709178 408106 214960 2902936 8711320 580544 3289650 2708884 2902235 2902476 1741437 967346
mismatch_points_fraction 8.230453e-05 0.01476373 0.02094759 0.9110513 0.3109869 0.04684654 0.02467529 0.3332284 0.9999736 0.06664072 0.3776194 0.3109532 0.3331479 0.3331756 0.1998998 0.1110418
num_data_points 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550 8711550
perfect_match_count 8710833 8582935 8529064 774881 6002372 8303444 8496590 5808614 230 8131006 5421900 6002666 5809315 5809074 873 7744204
perfect_match_fraction 0.9999177 0.9852363 0.9790524 0.2354524 0.6890131 0.9531535 0.9753247 0.6819255 0.0001110353 0.9333593 0.6668364 0.6890468 0.6820078 0.6840524 0.0005010589 0.8889582
r-squared correlation 0.98399 1 1 0.999999 0.999999 1 1 0.999999 0.999999 1 1 1 1 1 1 1
rms_val 2.401615 0.0002754752 0.0003371867 0.009954174 0.006200183 0.0007031779 0.0004275732 0.005983867 0.01363906 0.001328299 0.002640714 0.002669068 0.002106088 0.002128381 0.003741274 0.000858054
shape (45, 30, 9, 717) (45, 30, 9, 717) (45, 30, 9, 717) (45, 30, 9, 717) (45, 30, 9, 717) (45, 30, 9, 717) (45, 30, 9, 717) (45, 30, 9, 717) (45, 30, 9, 717) (45, 30, 9, 717) (45, 30, 9, 717) (45, 30, 9, 717) (45, 30, 9, 717) (45, 30, 9, 717) (45, 30, 9, 717) (45, 30, 9, 717)
spatially_invalid_pts_ignored_a 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
spatially_invalid_pts_ignored_b 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
std_val 2.40161 0.000275474 0.000337185 0.00959823 0.00619563 0.00070316 0.000427512 0.0059836 0.0126351 0.00132535 0.00263869 0.00266849 0.0020995 0.00211881 0.0037227 0.000857058
std_val_a 18.8277 20.6617 14.8977 13.784 18.888 27.0657 24.7929 18.9274 11.5147 15.8858 13.0027 15.6138 15.1701 14.4666 11.8862 14.3243
std_val_b 18.9803 20.6617 14.8977 14.9586 18.888 27.0657 24.7929 18.8318 12.846 15.8858 13.068 15.6138 15.1579 14.4262 11.8861 14.3243
In [18]:
df.rad_mw[df.rad_mw['max_diff'] > 0.0001].transpose()
Out[18]:
2016j004g119 2016j004g120 2016j028g037 2016j028g038 2016j028g041 2016j028g042 2016j028g052 2016j028g053 2016j028g054 2016j010g026 2016j010g027 2016j010g059 2016j010g060 2016j017g089 2016j017g090
a_finite_count 5309550 5309550 2005830 5309550 5309550 5309550 5191560 1262493 5309550 4955580 5309550 5191560 5175828 1061910 5309550
a_finite_fraction 1 1 0.3777778 1 1 1 0.9777778 0.2377778 1 0.9333333 1 0.9777778 0.9748148 0.2 1
a_missing_count 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
a_missing_fraction 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
a_missing_value 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36
a_nan_count 0 0 3303720 0 0 0 117990 4047057 0 353970 0 117990 133722 4247640 0
a_nan_fraction 0 0 0.6222222 0 0 0 0.02222222 0.7622222 0 0.06666667 0 0.02222222 0.02518519 0.8 0
b_finite_count 5309550 5309550 5309550 5309550 5309550 5309550 5309550 5309550 5309550 5309550 5309550 5309550 5309550 1061910 5309550
b_finite_fraction 1 1 1 1 1 1 1 1 1 1 1 1 1 0.2 1
b_missing_count 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
b_missing_fraction 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
b_missing_value 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36
b_nan_count 0 0 0 0 0 0 0 0 0 0 0 0 0 4247640 0
b_nan_fraction 0 0 0 0 0 0 0 0 0 0 0 0 0 0.8 0
common_finite_count 5309550 5309550 2005830 5309550 5309550 5309550 5191560 1262493 5309550 4955580 5309550 5191560 5175828 1061910 5309550
common_finite_fraction 1 1 0.3777778 1 1 1 0.9777778 0.2377778 1 0.9333333 1 0.9777778 0.9748148 0.2 1
common_missing_count 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
common_missing_fraction 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
common_nan_count 0 0 0 0 0 0 0 0 0 0 0 0 0 4247640 0
common_nan_fraction 0 0 0 0 0 0 0 0 0 0 0 0 0 0.8 0
correlation 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
diff_outside_epsilon_count 504301 825317 1533753 1651659 497855 261932 1651590 1262442 353921 1651497 1651519 1651394 1635832 1061603 589697
diff_outside_epsilon_fraction 0.09497999 0.1554401 0.7646476 0.3110733 0.09376595 0.04933224 0.3181298 0.9999596 0.06665744 0.3332601 0.3110469 0.3180921 0.3160522 0.9997109 0.1110635
epsilon 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
epsilon_percent NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
finite_in_only_one_count 0 0 3303720 0 0 0 117990 4047057 0 353970 0 117990 133722 0 0
finite_in_only_one_fraction 0 0 0.6222222 0 0 0 0.02222222 0.7622222 0 0.06666667 0 0.02222222 0.02518519 0 0
max_a 61.3342 52.365 24.7826 41.0498 60.5076 60.9204 48.9895 21.1842 38.1551 31.885 39.3073 36.7709 36.4703 21.8567 32.3465
max_b 61.3342 52.3656 32.4571 41.0498 60.5076 60.9204 48.9895 25.2821 38.1551 31.885 39.3073 36.7709 36.4703 21.8581 32.3465
max_delta 0.00790787 0.00744247 0.0280352 0.0345089 0.00356674 0.00260544 0.0389457 0.0377007 0.010363 0.00994349 0.0089969 0.00704062 0.00688547 0.00767064 0.00568199
max_diff 0.00790787 0.00744247 0.0280352 0.0345089 0.00372696 0.00361633 0.0389457 0.0395795 0.0123655 0.0120977 0.0126288 0.00867367 0.00872275 0.00807571 0.00568199
mean_a 10.2496 9.12577 4.94082 6.47687 13.4441 7.76521 6.92283 3.94008 5.98016 5.45137 5.91256 6.66429 5.57568 4.83982 5.82526
mean_b 10.2496 9.12577 5.61226 6.47701 13.4441 7.76521 6.90659 4.878 5.98014 5.49922 5.91256 6.6468 5.57981 4.83977 5.82525
mean_delta 7.4449e-06 -2.41988e-06 0.00053806 0.000139444 5.22066e-06 -2.52079e-06 0.000146378 -0.000953225 -2.09322e-05 3.45676e-05 5.34587e-06 3.86483e-05 -5.28558e-05 -4.73384e-05 -6.57078e-06
mean_diff 3.87522e-05 5.46452e-05 0.00175965 0.000755343 4.11956e-05 1.21278e-05 0.000680421 0.00310454 7.25874e-05 0.000319435 0.000312292 0.000235065 0.000243016 0.000474216 3.74566e-05
median_a 5.39023 5.70884 3.09512 3.86418 8.07736 4.26774 4.32275 2.5663 3.70098 3.58267 3.62907 4.42054 3.62078 3.45859 3.93761
median_b 5.39022 5.70884 3.48728 3.86431 8.07741 4.26775 4.31774 3.22555 3.70097 3.62002 3.62912 4.4107 3.62477 3.45858 3.9376
median_delta 0 0 0 0 0 0 0 -0.000797749 0 0 0 0 0 -2.90871e-05 0
median_diff 0 0 0.00101423 0 0 0 0 0.00231314 0 0 0 0 0 0.000324249 0
min_a 0.0955297 0.478158 0.154194 0.172566 0.360217 0.159565 0.372425 0.165025 0.170142 0.231368 0.158633 0.234396 0.146806 0.125121 0.123862
min_b 0.0955297 0.478101 0.165184 0.190521 0.360217 0.159565 0.372425 0.15566 0.170142 0.228707 0.158633 0.233119 0.151415 0.125111 0.123862
min_delta -0.00595856 -0.00614929 -0.0277494 -0.0308228 -0.00372696 -0.00361633 -0.034391 -0.0395795 -0.0123655 -0.0120977 -0.0126288 -0.00867367 -0.00872275 -0.00807571 -0.00473404
mismatch_points_count 504301 825317 4837473 1651659 497855 261932 1769580 5309499 353921 2005467 1651519 1769384 1769554 1061603 589697
mismatch_points_fraction 0.09497999 0.1554401 0.9110891 0.3110733 0.09376595 0.04933224 0.3332825 0.9999904 0.06665744 0.3777094 0.3110469 0.3332456 0.3332776 0.1999422 0.1110635
num_data_points 5309550 5309550 5309550 5309550 5309550 5309550 5309550 5309550 5309550 5309550 5309550 5309550 5309550 5309550 5309550
perfect_match_count 4805249 4484233 472077 3657891 4811695 5047618 3539970 51 4955629 3304083 3658031 3540166 3539996 307 4719853
perfect_match_fraction 0.90502 0.8445599 0.2353524 0.6889267 0.906234 0.9506678 0.6818702 4.039626e-05 0.9333426 0.6667399 0.6889531 0.6819079 0.6839478 0.0002891017 0.8889365
r-squared correlation 1 1 1 1 1 1 1 0.999999 1 1 1 0.999999 1 1 1
rms_val 0.0001891915 0.000211838 0.002864956 0.001968659 0.0001887693 8.175842e-05 0.001783236 0.004294648 0.0003896906 0.000776016 0.00076885 0.000571272 0.0005834488 0.0007022137 0.0001661747
shape (45, 30, 9, 437) (45, 30, 9, 437) (45, 30, 9, 437) (45, 30, 9, 437) (45, 30, 9, 437) (45, 30, 9, 437) (45, 30, 9, 437) (45, 30, 9, 437) (45, 30, 9, 437) (45, 30, 9, 437) (45, 30, 9, 437) (45, 30, 9, 437) (45, 30, 9, 437) (45, 30, 9, 437) (45, 30, 9, 437)
spatially_invalid_pts_ignored_a 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
spatially_invalid_pts_ignored_b 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
std_val 0.000189045 0.000211824 0.00281398 0.00196373 0.000188697 8.17196e-05 0.00177722 0.00418752 0.000389126 0.000775246 0.000768832 0.000569963 0.00058105 0.000700616 0.000166046
std_val_a 10.9977 8.60006 4.41107 6.35029 12.9196 8.72693 6.71854 3.51678 5.47226 4.6068 5.40332 5.65437 4.82373 3.74852 4.88726
std_val_b 10.9977 8.60005 5.05222 6.35031 12.9196 8.72693 6.68894 4.14066 5.47226 4.63859 5.40331 5.6361 4.81983 3.74847 4.88726
In [19]:
df.rad_sw[df.rad_sw['max_diff'] > 0.00001].transpose()
Out[19]:
2016j004g119 2016j004g120 2016j025g202 2016j028g037 2016j028g038 2016j028g052 2016j028g053 2016j028g054 2016j010g026 2016j010g027 2016j010g059 2016j010g060 2016j017g089 2016j017g090
a_finite_count 1980450 1980450 1980450 748170 1980450 1936440 471559 1980450 1848420 1980450 1936440 1930572 396090 1980450
a_finite_fraction 1 1 1 0.3777778 1 0.9777778 0.238107 1 0.9333333 1 0.9777778 0.9748148 0.2 1
a_missing_count 0 0 0 0 0 0 0 0 0 0 0 0 0 0
a_missing_fraction 0 0 0 0 0 0 0 0 0 0 0 0 0 0
a_missing_value 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36
a_nan_count 0 0 0 1232280 0 44010 1508891 0 132030 0 44010 49878 1584360 0
a_nan_fraction 0 0 0 0.6222222 0 0.02222222 0.761893 0 0.06666667 0 0.02222222 0.02518519 0.8 0
b_finite_count 1980450 1980450 1980450 1980450 1980450 1980450 1980450 1980450 1980450 1980450 1980450 1980450 396090 1980450
b_finite_fraction 1 1 1 1 1 1 1 1 1 1 1 1 0.2 1
b_missing_count 0 0 0 0 0 0 0 0 0 0 0 0 0 0
b_missing_fraction 0 0 0 0 0 0 0 0 0 0 0 0 0 0
b_missing_value 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36 9.96921e+36
b_nan_count 0 0 0 0 0 0 0 0 0 0 0 0 1584360 0
b_nan_fraction 0 0 0 0 0 0 0 0 0 0 0 0 0.8 0
common_finite_count 1980450 1980450 1980450 748170 1980450 1936440 471559 1980450 1848420 1980450 1936440 1930572 396090 1980450
common_finite_fraction 1 1 1 0.3777778 1 0.9777778 0.238107 1 0.9333333 1 0.9777778 0.9748148 0.2 1
common_missing_count 0 0 0 0 0 0 0 0 0 0 0 0 0 0
common_missing_fraction 0 0 0 0 0 0 0 0 0 0 0 0 0 0
common_nan_count 0 0 0 0 0 0 0 0 0 0 0 0 1584360 0
common_nan_fraction 0 0 0 0 0 0 0 0 0 0 0 0 0.8 0
correlation 1 1 1 0.999996 0.999999 1 0.99998 1 1 1 1 1 1 1
diff_outside_epsilon_count 107543 175973 70809 572116 616124 616090 471554 132024 616109 616118 616090 610248 396051 220012
diff_outside_epsilon_fraction 0.05430231 0.08885506 0.035754 0.7646872 0.311103 0.318156 0.9999894 0.06666364 0.3333166 0.3111 0.318156 0.316097 0.9999015 0.1110919
epsilon 0 0 0 0 0 0 0 0 0 0 0 0 0 0
epsilon_percent NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
finite_in_only_one_count 0 0 0 1232280 0 44010 1508891 0 132030 0 44010 49878 0 0
finite_in_only_one_fraction 0 0 0 0.6222222 0 0.02222222 0.761893 0 0.06666667 0 0.02222222 0.02518519 0 0
max_a 3.51303 2.6383 3.30614 0.705122 1.66522 2.29703 0.54858 1.4558 1.07193 1.51985 1.89978 1.33886 0.563671 1.10456
max_b 3.51303 2.63848 3.30612 1.12611 1.66522 2.29703 0.744842 1.4558 1.07193 1.51985 1.89978 1.33886 0.56363 1.10456
max_delta 0.000519991 0.000458479 0.000315681 0.00158426 0.00208792 0.00193238 0.00212775 0.000569738 0.000490278 0.00051403 0.000407398 0.00030913 0.000233173 0.000144869
max_diff 0.000557303 0.000522196 0.000315681 0.00173483 0.00208792 0.00193238 0.00223789 0.000569738 0.000509387 0.00051403 0.000407398 0.00040473 0.00025782 0.000144869
mean_a 0.560314 0.341762 0.278872 0.100926 0.170854 0.243837 0.0888711 0.136731 0.119584 0.133658 0.172129 0.116653 0.078787 0.111377
mean_b 0.560315 0.341762 0.278872 0.118891 0.170874 0.241952 0.109794 0.13673 0.119631 0.133657 0.171008 0.116767 0.0787866 0.111377
mean_delta 5.87823e-07 3.22659e-07 -1.44577e-07 2.00765e-05 2.04525e-05 2.37598e-05 -2.19642e-05 -4.44853e-07 -2.54903e-06 -1.37323e-06 2.95425e-06 -1.09283e-06 -3.17961e-07 -1.01463e-07
mean_diff 2.83228e-06 2.65236e-06 1.76758e-06 0.000155227 7.87763e-05 6.55374e-05 0.000311976 6.5059e-06 3.00498e-05 3.01748e-05 1.98516e-05 2.12983e-05 1.77533e-05 1.28777e-06
median_a 0.45487 0.228614 0.115785 0.0765986 0.103831 0.162785 0.0752243 0.0959955 0.0930657 0.0918929 0.127524 0.0824355 0.057655 0.0749106
median_b 0.454871 0.228614 0.115785 0.0847282 0.103851 0.161702 0.0883191 0.0959946 0.0928289 0.0918914 0.1268 0.082862 0.0576556 0.0749109
median_delta 0 0 0 0 0 0 -2.45329e-05 0 0 0 0 0 -1.22935e-07 0
median_diff 0 0 0 9.49502e-05 0 0 0.00024461 0 0 0 0 0 1.17533e-05 0
min_a -0.00619526 0.0091627 0.00862566 0.00374 0.000262607 0.0237659 0.00811927 -0.000492206 0.00735278 0.00232111 0.0134761 0.00497359 0.00149788 -0.00209385
min_b -0.00619526 0.0091627 0.00862566 0.0017583 0.000262607 0.0236012 0.00846421 -0.000492206 0.00527232 0.00243447 0.0106614 0.00497359 0.00149802 -0.00209385
min_delta -0.000557303 -0.000522196 -0.000285149 -0.00173483 -0.00182804 -0.00111239 -0.00223789 -0.000558384 -0.000509387 -0.000482712 -0.000329196 -0.00040473 -0.00025782 -0.000139594
mismatch_points_count 107543 175973 70809 1804396 616124 660100 1980445 132024 748139 616118 660100 660126 396051 220012
mismatch_points_fraction 0.05430231 0.08885506 0.035754 0.911104 0.311103 0.3333081 0.9999975 0.06666364 0.3777621 0.3111 0.3333081 0.3333212 0.1999803 0.1110919
num_data_points 1980450 1980450 1980450 1980450 1980450 1980450 1980450 1980450 1980450 1980450 1980450 1980450 1980450 1980450
perfect_match_count 1872907 1804477 1909641 176054 1364326 1320350 5 1848426 1232311 1364332 1320350 1320324 39 1760438
perfect_match_fraction 0.9456977 0.9111449 0.964246 0.2353128 0.688897 0.681844 1.060313e-05 0.9333364 0.6666834 0.6889 0.681844 0.683903 9.846247e-05 0.8889081
r-squared correlation 1 1 1 0.999992 0.999998 0.999999 0.99996 1 0.999999 1 1 1 0.999999 1
rms_val 1.954852e-05 1.511002e-05 1.323422e-05 0.0002392996 0.000193207 0.0001627942 0.0004080512 3.290218e-05 6.754574e-05 6.844076e-05 4.53313e-05 4.784354e-05 2.590639e-05 5.619698e-06
shape (45, 30, 9, 163) (45, 30, 9, 163) (45, 30, 9, 163) (45, 30, 9, 163) (45, 30, 9, 163) (45, 30, 9, 163) (45, 30, 9, 163) (45, 30, 9, 163) (45, 30, 9, 163) (45, 30, 9, 163) (45, 30, 9, 163) (45, 30, 9, 163) (45, 30, 9, 163) (45, 30, 9, 163)
spatially_invalid_pts_ignored_a 0 0 0 0 0 0 0 0 0 0 0 0 0 0
spatially_invalid_pts_ignored_b 0 0 0 0 0 0 0 0 0 0 0 0 0 0
std_val 1.95397e-05 1.51065e-05 1.32334e-05 0.000238456 0.000192122 0.000161051 0.00040746 3.28992e-05 6.74976e-05 6.84272e-05 4.52349e-05 4.78311e-05 2.59044e-05 5.61881e-06
std_val_a 0.558456 0.326407 0.358192 0.0874736 0.201462 0.233709 0.0639245 0.138548 0.0923064 0.134306 0.145101 0.108723 0.0659062 0.109517
std_val_b 0.558457 0.326408 0.358192 0.114773 0.201464 0.231992 0.0774253 0.138547 0.0931181 0.134306 0.144238 0.108215 0.065906 0.109516
In [20]:
import netCDF4 as nc4 
from glob import glob
afn, = list(glob('A/2016/005/S*g044.L1B*nc'))
gfn, = list(glob('G/2016/005/S*g044.L1B*nc'))

A, G = nc4.Dataset(afn), nc4.Dataset(gfn)
ais = A['instrument_state']
gis = G['instrument_state']
In [21]:
%matplotlib inline
import netCDF4 as nc4
from numpy import *
from pylab import *
def fig(a=20, b=20, **kwargs):
    return figure(figsize=(a,b), **kwargs)

/home/rayg/.conda/envs/cris/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.
  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')
In [22]:
np.argwhere(ais[:]!=gis[:])
Out[22]:
array([[14, 22,  7]])
In [23]:
A['l1b_qual'][14,22,7], G['l1b_qual'][14,22,7]
Out[23]:
(0, 524290)
In [24]:
arad = A['rad_lw']
srad = G['rad_lw']
figure(figsize=(20,20))
plot(arad[14, 22, 7])
plot(srad[14, 22, 7])
legend(('A-SIPS', 'GES-DISC'))
grid()
title('2016j005g044 rad_lw[14,22,7]')
Out[24]:
<matplotlib.text.Text at 0x7ffa2001ee10>
In [25]:
# use cksum to verify that download was correct, compared to CRC32 in XML metadata file
# (cris)[fourier:020]: cksum SNDR.SNPP.CRIS.20160120T2148.m06.g219.L1B_NSR.std.v01_08.G.161224201254.nc 
# 1640188193 130672735 SNDR.SNPP.CRIS.20160120T2148.m06.g219.L1B_NSR.std.v01_08.G.161224201254.nc