From 0190f0bfd0378a9e3ec954af6b579c68923d9e52 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 19 Sep 2018 14:13:55 +0100 Subject: [PATCH 001/103] updates --- MANIFEST.in | 3 ++- README.md | 15 ++++++++++++++ SIAC/SIAC_L8.py | 3 ++- SIAC/SIAC_S2.py | 3 ++- SIAC/data/.earthdata_auth | 0 SIAC/get_MCD43.py | 43 ++++++++++++++++++++++++--------------- SIAC/kernels.py | 15 +++++++++----- SIAC/lhd.py | 9 ++++---- SIAC/smoothn.py | 5 +++-- SIAC/the_aerosol.py | 1 - SIAC/the_correction.py | 1 - requirements.txt | 6 +++--- setup.py | 17 ++++++++++------ tests/test_dependency.py | 40 ++++++++++++++++++++++++++++++++++++ tests/test_uni.py | 18 ++++++++++++++++ 15 files changed, 138 insertions(+), 41 deletions(-) create mode 100644 SIAC/data/.earthdata_auth create mode 100644 tests/test_dependency.py create mode 100644 tests/test_uni.py diff --git a/MANIFEST.in b/MANIFEST.in index 1b58732..523cabd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include setup.py include SIAC/util/* -include SIAC/emus/*.pkl include SIAC/data/sen2cloud_detector.pkl +include SIAC/data/.earthdata_auth include SIAC/spectral_mapping/* +recursive-include tests *.py diff --git a/README.md b/README.md index ad8da84..6cef536 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,16 @@ This atmospheric correction method uses MODIS MCD43 BRDF product to get a coarse Download this repositories either by `git clone git@ github.com:MarcYin/SIAC.git` or download the zip file and unzip it. In the main directory of it excute `pip install .` +To save your time for installing GDAL and lightgbm, please use: + +```bash +conda install -c conda-forge gdal=2.2.4 +conda install -c conda-forge lightgbm +``` + +or `pip install SIAC` or `conda install -c f0xy siac` + + The typical usage for Sentinel 2 and Landsat 8: ```python from SIAC import SIAC_S2 @@ -29,6 +39,11 @@ SIAC_L8('/directory/where/you/store/L8/data/') An example of correction for Landsat 5 for a more detailed demostration of the usage is shown [here](https://github.com/MarcYin/Global-analysis-ready-dataset) +## Examples and Map: + +A [page](http://www2.geog.ucl.ac.uk/~ucfafyi/Atmo_Cor/index.html) shows some correction samples. + +A [map](http://www2.geog.ucl.ac.uk/~ucfafyi/map) for comparison between TOA and BOA. ### LICENSE GNU GENERAL PUBLIC LICENSE V3 diff --git a/SIAC/SIAC_L8.py b/SIAC/SIAC_L8.py index 0292118..16970f6 100644 --- a/SIAC/SIAC_L8.py +++ b/SIAC/SIAC_L8.py @@ -15,7 +15,7 @@ home = expanduser("~") file_path = os.path.dirname(os.path.realpath(__file__)) -def SIAC_L8(l8_dir, send_back = False): +def SIAC_L8(l8_dir, send_back = False, mcd43 = home + '/MCD43', vrt_dir = home + '/MCD43_VRT'): file_path = os.path.dirname(os.path.realpath(__file__)) if not os.path.exists(file_path + '/emus/'): os.mkdir(file_path + '/emus/') @@ -35,6 +35,7 @@ def SIAC_L8(l8_dir, send_back = False): rets = l8_pre_processing(l8_dir) aero_atmos = [] for ret in rets: + ret += (mcd43, vrt_dir) #sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile = ret aero_atmo = do_correction(*ret) if send_back: diff --git a/SIAC/SIAC_S2.py b/SIAC/SIAC_S2.py index 724aa38..06764a3 100644 --- a/SIAC/SIAC_S2.py +++ b/SIAC/SIAC_S2.py @@ -14,7 +14,7 @@ home = expanduser("~") file_path = os.path.dirname(os.path.realpath(__file__)) -def SIAC_S2(s2_t, send_back = False): +def SIAC_S2(s2_t, send_back = False, mcd43 = home + '/MCD43', vrt_dir = home + '/MCD43_VRT'): if not os.path.exists(file_path + '/emus/'): os.mkdir(file_path + '/emus/') if len(glob(file_path + '/emus/' + 'isotropic_MSI_emulators_*_x?p_S2?.pkl')) < 12: @@ -31,6 +31,7 @@ def SIAC_S2(s2_t, send_back = False): rets = s2_pre_processing(s2_t) aero_atmos = [] for ret in rets: + ret += (mcd43, vrt_dir) #sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile = ret aero_atmo = do_correction(*ret) if send_back: diff --git a/SIAC/data/.earthdata_auth b/SIAC/data/.earthdata_auth new file mode 100644 index 0000000..e69de29 diff --git a/SIAC/get_MCD43.py b/SIAC/get_MCD43.py index 9438b47..6639212 100644 --- a/SIAC/get_MCD43.py +++ b/SIAC/get_MCD43.py @@ -1,6 +1,7 @@ import os import time import gdal +import getpass import logging import requests import numpy as np @@ -11,26 +12,36 @@ from SIAC.modis_tile_cal import get_vector_hv, get_raster_hv from os.path import expanduser from SIAC.create_logger import create_logger +from six.moves import input + home = expanduser("~") -''' -logger = logging.getLogger('SIAC') -logger.setLevel(logging.INFO) -if not logger.handlers: - ch = logging.StreamHandler() - ch.setLevel(logging.DEBUG) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - ch.setFormatter(formatter) - logger.addHandler(ch) -''' +file_path = os.path.dirname(os.path.realpath(__file__)) logger = create_logger() -try: - username, password = np.loadtxt(home + '/earthdata_auth', dtype=str) - auth = tuple([username, password]) -except: - username, password = np.loadtxt('earthdata_auth', dtype=str) - auth = tuple([username, password]) +if os.path.exists(file_path + '/data/.earthdata_auth'): + try: + username, password = np.loadtxt(file_path + '/data/.earthdata_auth', dtype=str) + auth = tuple([username, password]) + except: + logger.error('Please provide NASA Earthdata username and password for downloading MCD43 data, which can be applied here: https://urs.earthdata.nasa.gov.') + username = input('Username for NASA Earthdata: ') + password = getpass.getpass('Password for NASA Earthdata: ') + up = username, password + os.remove(file_path + '/data/.earthdata_auth') + with open(file_path + '/data/.earthdata_auth', 'wb') as f: + for i in up: + f.write((i + '\n').encode()) + auth = tuple([username, password]) +else: + username = input('Username for NASA Earthdata: ') + password = getpass.getpass('Password for NASA Earthdata: ') + up = username, password + with open(home + '/.earthdata_auth', 'wb') as f: + for i in up: + f.write((i + '\n').encode()) + auth = tuple([username, password]) + def find_files(aoi, obs_time, temporal_window = 16): days = [(obs_time - timedelta(days = int(i))).strftime('%Y.%m.%d') for i in np.arange(temporal_window, 0, -1)] + \ [(obs_time + timedelta(days = int(i))).strftime('%Y.%m.%d') for i in np.arange(0, temporal_window+1, 1)] diff --git a/SIAC/kernels.py b/SIAC/kernels.py index c9d3ea9..51d1f64 100644 --- a/SIAC/kernels.py +++ b/SIAC/kernels.py @@ -583,6 +583,7 @@ def readPOLDER(inputFile,type=1): REFL[j,i] = float(s[j+6]) return VZA,SZA,RAA,REFL +''' def legend(*args, **kwargs): """ Overwrites the pylab legend function. @@ -594,7 +595,7 @@ def legend(*args, **kwargs): from http://www.mail-archive.com/matplotlib-users@lists.sourceforge.net/msg04256.html """ - import pylab + #import pylab if kwargs.has_key('loc'): loc = kwargs['loc'] loc = loc.split() @@ -633,7 +634,7 @@ def legend(*args, **kwargs): return leg return pylab.legend(*args, **kwargs) - +''' def lutInvertRossHS(VZA,SZA,RAA,REFL,N=1000,fixXi=False,RossType='Thick',LiType='Dense',normalise=1,RecipFlag=True,MODISSPARSE=True): @@ -779,8 +780,10 @@ def testLisa(inputFile,buff=30,LiType='Sparse',RossType='Thick',plot=False,verbo if nbands == 4: return doy,minrmse,minP,minFWD,minrefl,mindoy,sig[mincount],rel[mincount] else: + pass + ''' if plot: - import pylab + #import pylab x = [mindoy,mindoy] y = [0.0,max(numpy.array([minFWD.flatten(),minrefl.flatten()]).flatten())+0.1] pylab.plot(x,y) @@ -808,6 +811,7 @@ def testLisa(inputFile,buff=30,LiType='Sparse',RossType='Thick',plot=False,verbo pylab.plot(bands,maxfcc*maxsBurn,'rD',label='fcc * burn signal') pylab.legend(loc=0) pylab.show() + ''' return doy,minrmse,minP,minFWD,minrefl,mindoy,sig[mincount],rel[mincount],maxfcc,maxa0,maxa1 else: if nbands == 4: @@ -986,8 +990,9 @@ def mimic(doPrint=False,doPlot=False,RossHS=False,RecipFlag=False,thisSza=None): if doPrint == True: kk.printKernels(file='RossThinLiTransit.' + str(SZAS[i]) + '.dat') kk.printer('') + ''' if (doPlot == True): - import pylab + #import pylab x = [-90.0,90.0] y = [0.0,0.0] for i in range(len(SZAS)): @@ -1006,5 +1011,5 @@ def mimic(doPrint=False,doPlot=False,RossHS=False,RecipFlag=False,thisSza=None): pylab.axis([-90.0,90.0,-3.0,3.0]) pylab.legend(loc=0) pylab.show() - + ''' return VZA,SZA,RAA,RossThick,RossThin,LiSparse,LiDense,Roujean,LiTransit diff --git a/SIAC/lhd.py b/SIAC/lhd.py index 5cceb2b..24f9b88 100644 --- a/SIAC/lhd.py +++ b/SIAC/lhd.py @@ -285,10 +285,11 @@ def fill_space(data): rand_lhs = lhd(dist=(d0,d1,d2,d3),size=100) spac_lhs = lhd(dist=(d0,d1,d2,d3),size=100,form='spacefilling', iterations=100,showcorrelations=True) - + ''' try: - from scatterplot_matrix import scatterplot_matrix as spm - import matplotlib.pyplot as plt + #from scatterplot_matrix import scatterplot_matrix as spm + #import matplotlib.pyplot as plt + except ImportError: print (rand_lhs) print (spac_lhs) @@ -300,4 +301,4 @@ def fill_space(data): spm(spac_lhs.T,names=names) plt.suptitle('Space-Filling LHS Design') plt.show() - + ''' diff --git a/SIAC/smoothn.py b/SIAC/smoothn.py index 73c8c9b..a1bf7a9 100644 --- a/SIAC/smoothn.py +++ b/SIAC/smoothn.py @@ -1,7 +1,7 @@ from numpy import * -from pylab import * import scipy.optimize.lbfgsb as lbfgsb import numpy.linalg +from numpy.linalg import norm from scipy.fftpack.realtransforms import dct,idct import numpy as np import numpy.ma as ma @@ -531,6 +531,7 @@ def peaks(n): z += f return z +''' def test1(): plt.figure(1) plt.clf() @@ -711,4 +712,4 @@ def sparseTest(n=1000): # ((Vt.T * (np.diag(np.array(eigenvalues).flatten())**2)) * Vt) # we see you get the same as m.T * m by squaring the eigenvalues - +''' diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index 148cc94..b7940b7 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -4,7 +4,6 @@ import os import gc import sys -#sys.path.insert(0, '/home/ucfafyi/DATA/Atmospheric_correction/atmospheric_correction/util') import ogr import osr import gdal diff --git a/SIAC/the_correction.py b/SIAC/the_correction.py index e4b2f91..e9c64f6 100644 --- a/SIAC/the_correction.py +++ b/SIAC/the_correction.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import os import sys -#sys.path.insert(0,'/home/ucfafyi/DATA/Atmospheric_correction/atmospheric_correction/util') import gdal import psutil import logging diff --git a/requirements.txt b/requirements.txt index 4b03121..846b42e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -pandas +six psutil requests scikit-image scikit-learn -GDAL>=2.1 -gp-emulator==1.5.5 +GDAL>=2.1, <2.3 +gp_emulator>=1.5.5 lightgbm>=2.1.0 numpy>=1.13 scipy>=1.0.0 diff --git a/setup.py b/setup.py index ad597ec..b0e6f6e 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,19 @@ from setuptools import setup + +with open('README.md', 'r', 'utf-8') as f: + readme = f.read() + setup(name ='SIAC', - version ='2.0', + version ='2.0.0', description = 'A sensor invariant Atmospheric Correction (SIAC)', + long_description = readme, author = 'Feng Yin', author_email = 'ucfafyi@ucl.ac.uk', - classifiers = ['Development Status :: 2 - BETA', - 'Programming Language :: Python :: 2.7, 3.6', - 'Topic :: Atmospheric Correction'], - install_requires = ['gdal>=2.1', 'numpy>=1.13', 'scipy>=1.0', 'lightgbm>=2.1.0',\ - 'requests', 'scikit-learn', 'scikit-image', 'pandas', 'psutil'], + classifiers = ['Development Status :: 4 - Beta', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6'], + install_requires = ['gdal>=2.1, <2.3', 'numpy>=1.13', 'scipy>=1.0', 'psutil','six', + 'lightgbm>=2.1.0','requests', 'scikit-learn', 'scikit-image', 'gp_emulator>=1.6.0'], url = 'https://github.com/MarcYin/SIAC', license = "GNU Affero General Public License v3.0", include_package_data = True, diff --git a/tests/test_dependency.py b/tests/test_dependency.py new file mode 100644 index 0000000..9274eda --- /dev/null +++ b/tests/test_dependency.py @@ -0,0 +1,40 @@ +import os +from os.path import expanduser +home = expanduser("~") + +def test_sen2cloud(): + sen2cloud_path = file_path = os.path.dirname(os.path.realpath(__file__)) + assert os.path.exists(file_path.replace('tests', 'SIAC/data/sen2cloud_detector.pkl')) + +def test_earthdata_auth(): + sen2cloud_path = file_path = os.path.dirname(os.path.realpath(__file__)) + assert os.path.exists(file_path.replace('tests', 'SIAC/data/.earthdata_auth')) + +def test_spectral_mapping_1(): + sen2cloud_path = file_path = os.path.dirname(os.path.realpath(__file__)) + assert os.path.exists(file_path.replace('tests', 'SIAC/spectral_mapping/TERRA_S2A_spectral_mapping.txt')) + +def test_spectral_mapping_2(): + sen2cloud_path = file_path = os.path.dirname(os.path.realpath(__file__)) + assert os.path.exists(file_path.replace('tests', 'SIAC/spectral_mapping/TERRA_S2B_spectral_mapping.txt')) + +def test_spectral_mapping_3(): + sen2cloud_path = file_path = os.path.dirname(os.path.realpath(__file__)) + assert os.path.exists(file_path.replace('tests', 'SIAC/spectral_mapping/TERRA_L8_spectral_mapping.txt')) + +def test_spectral_mapping_4(): + sen2cloud_path = file_path = os.path.dirname(os.path.realpath(__file__)) + assert os.path.exists(file_path.replace('tests', 'SIAC/spectral_mapping/TERRA_TERRA_spectral_mapping.txt')) +def test_l8_angle(): + sen2cloud_path = file_path = os.path.dirname(os.path.realpath(__file__)) + assert os.path.exists(file_path.replace('tests', 'SIAC/util/l8_angles')) + + +if __name__ == '__main__': + test_sen2cloud() + test_earthdata_auth() + test_spectral_mapping_1() + test_spectral_mapping_2() + test_spectral_mapping_3() + test_spectral_mapping_4() + test_l8_angle() diff --git a/tests/test_uni.py b/tests/test_uni.py new file mode 100644 index 0000000..6a09316 --- /dev/null +++ b/tests/test_uni.py @@ -0,0 +1,18 @@ +def test_import_SIAC_S2(): + try: + from SIAC import SIAC_S2 + ret = True + except: + ret = False + assert ret + +def test_import_SIAC_L8(): + try: + from SIAC import SIAC_L8 + ret = True + except: + ret = False + assert ret +if __name__ == '__main__': + test_import_SIAC_S2() + test_import_SIAC_L8() From 864de230b6b3af52e5873a2f58107a3925ae4c85 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 19 Sep 2018 14:16:29 +0100 Subject: [PATCH 002/103] updates --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 6cef536..9596584 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ This atmospheric correction method uses MODIS MCD43 BRDF product to get a coarse ## Installation: -Download this repositories either by `git clone git@ github.com:MarcYin/SIAC.git` or download the zip file and unzip it. In the main directory of it excute `pip install .` +Download this repositories either by `git clone git@ github.com:MarcYin/SIAC.git` or download the zip file and unzip it. In the main directory of it excute `pip install .` , or `pip install SIAC` or `conda install -c f0xy siac` To save your time for installing GDAL and lightgbm, please use: @@ -24,7 +24,6 @@ conda install -c conda-forge gdal=2.2.4 conda install -c conda-forge lightgbm ``` -or `pip install SIAC` or `conda install -c f0xy siac` The typical usage for Sentinel 2 and Landsat 8: From ee072f1fee5d99dd4fbdfad3a51b89eac9201526 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 19 Sep 2018 15:17:22 +0100 Subject: [PATCH 003/103] add toy test --- .gitignore | 1 + tests/test_dependency.py | 10 ++-------- tests/test_import.py | 15 +++++++++++++++ tests/test_uni.py | 18 ------------------ 4 files changed, 18 insertions(+), 26 deletions(-) create mode 100644 tests/test_import.py delete mode 100644 tests/test_uni.py diff --git a/.gitignore b/.gitignore index b8b4119..f70cbd0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*.pytest_cache* *.pyc *__pycache__* *.hdf diff --git a/tests/test_dependency.py b/tests/test_dependency.py index 9274eda..36e7c67 100644 --- a/tests/test_dependency.py +++ b/tests/test_dependency.py @@ -1,35 +1,29 @@ import os from os.path import expanduser home = expanduser("~") +file_path = os.path.dirname(os.path.realpath(__file__)) def test_sen2cloud(): - sen2cloud_path = file_path = os.path.dirname(os.path.realpath(__file__)) assert os.path.exists(file_path.replace('tests', 'SIAC/data/sen2cloud_detector.pkl')) def test_earthdata_auth(): - sen2cloud_path = file_path = os.path.dirname(os.path.realpath(__file__)) assert os.path.exists(file_path.replace('tests', 'SIAC/data/.earthdata_auth')) def test_spectral_mapping_1(): - sen2cloud_path = file_path = os.path.dirname(os.path.realpath(__file__)) assert os.path.exists(file_path.replace('tests', 'SIAC/spectral_mapping/TERRA_S2A_spectral_mapping.txt')) def test_spectral_mapping_2(): - sen2cloud_path = file_path = os.path.dirname(os.path.realpath(__file__)) assert os.path.exists(file_path.replace('tests', 'SIAC/spectral_mapping/TERRA_S2B_spectral_mapping.txt')) def test_spectral_mapping_3(): - sen2cloud_path = file_path = os.path.dirname(os.path.realpath(__file__)) assert os.path.exists(file_path.replace('tests', 'SIAC/spectral_mapping/TERRA_L8_spectral_mapping.txt')) def test_spectral_mapping_4(): - sen2cloud_path = file_path = os.path.dirname(os.path.realpath(__file__)) assert os.path.exists(file_path.replace('tests', 'SIAC/spectral_mapping/TERRA_TERRA_spectral_mapping.txt')) + def test_l8_angle(): - sen2cloud_path = file_path = os.path.dirname(os.path.realpath(__file__)) assert os.path.exists(file_path.replace('tests', 'SIAC/util/l8_angles')) - if __name__ == '__main__': test_sen2cloud() test_earthdata_auth() diff --git a/tests/test_import.py b/tests/test_import.py new file mode 100644 index 0000000..acb6439 --- /dev/null +++ b/tests/test_import.py @@ -0,0 +1,15 @@ +import sys, os +myPath = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, myPath + '/../') +user_pass = 'user', 'pass' + +with open(myPath.replace('tests', '') + '/SIAC/data/.earthdata_auth', 'wb') as f: + for i in user_pass: + f.write((i+'\n').encode()) + +from SIAC import SIAC_S2 +from SIAC import SIAC_L8 + + +with open(myPath.replace('tests', '') + '/SIAC/data/.earthdata_auth', 'wb') as f: + f.write(('').encode()) diff --git a/tests/test_uni.py b/tests/test_uni.py deleted file mode 100644 index 6a09316..0000000 --- a/tests/test_uni.py +++ /dev/null @@ -1,18 +0,0 @@ -def test_import_SIAC_S2(): - try: - from SIAC import SIAC_S2 - ret = True - except: - ret = False - assert ret - -def test_import_SIAC_L8(): - try: - from SIAC import SIAC_L8 - ret = True - except: - ret = False - assert ret -if __name__ == '__main__': - test_import_SIAC_S2() - test_import_SIAC_L8() From b9bcb51f8177a1680abcb53372049fe407ae9e70 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 19 Sep 2018 16:19:34 +0100 Subject: [PATCH 004/103] add test --- .travis.yml | 28 ++++++++++++++++++++++++++++ README.md | 2 ++ requirements.txt | 2 +- setup.py | 35 ++++++++++++++++++----------------- 4 files changed, 49 insertions(+), 18 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..f2471dd --- /dev/null +++ b/.travis.yml @@ -0,0 +1,28 @@ +language: python +python: + - "2.7" + - "3.6" +install: + - sudo apt-get update + - echo $TRAVIS_PYTHON_VERSION + - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then + wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; + else + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + fi + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda update -q conda + - conda info -a + - conda create -q -n test python=$TRAVIS_PYTHON_VERSION numpy numba scipy pytest pytest-cov coverage sphinx nose + - source activate test + - pip install -U codecov + - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then + pip install -U backports.functools_lru_cache; + fi + - python setup.py install + +script: + - pytest diff --git a/README.md b/README.md index 9596584..ea8e284 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ ### ucfafyi@ucl.ac.uk +[![PyPI version](https://badge.fury.io/py/SIAC.svg)](https://github.com/MarcYin/SIAC) + This atmospheric correction method uses MODIS MCD43 BRDF product to get a coarse resolution simulation of earth surface. A model based on MODIS PSF is built to deal with the scale differences between MODIS and Sentinel 2 / Landsat 8. We uses the ECMWF CAMS prediction as a prior for the atmospheric states, coupling with 6S model to solve for the atmospheric parameters. We do not have topography correction and homogeneouse surface is used without considering the BRDF effects. diff --git a/requirements.txt b/requirements.txt index 846b42e..4942247 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ requests scikit-image scikit-learn GDAL>=2.1, <2.3 -gp_emulator>=1.5.5 +gp_emulator>=1.6.1 lightgbm>=2.1.0 numpy>=1.13 scipy>=1.0.0 diff --git a/setup.py b/setup.py index b0e6f6e..49ecb61 100644 --- a/setup.py +++ b/setup.py @@ -1,21 +1,22 @@ from setuptools import setup -with open('README.md', 'r', 'utf-8') as f: - readme = f.read() +with open('README.md', 'rb') as f: + readme = f.read().decode() -setup(name ='SIAC', - version ='2.0.0', - description = 'A sensor invariant Atmospheric Correction (SIAC)', - long_description = readme, - author = 'Feng Yin', - author_email = 'ucfafyi@ucl.ac.uk', - classifiers = ['Development Status :: 4 - Beta', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6'], - install_requires = ['gdal>=2.1, <2.3', 'numpy>=1.13', 'scipy>=1.0', 'psutil','six', - 'lightgbm>=2.1.0','requests', 'scikit-learn', 'scikit-image', 'gp_emulator>=1.6.0'], - url = 'https://github.com/MarcYin/SIAC', - license = "GNU Affero General Public License v3.0", - include_package_data = True, - packages = ['SIAC'], +setup(name ='SIAC', + version ='2.0.2', + description = 'A sensor invariant Atmospheric Correction (SIAC)', + long_description = readme, + long_description_content_type ='text/markdown', + author = 'Feng Yin', + author_email = 'ucfafyi@ucl.ac.uk', + classifiers = ['Development Status :: 4 - Beta', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6'], + install_requires = ['gdal>=2.1, <2.3', 'numpy>=1.13', 'scipy>=1.0', 'psutil','six', + 'lightgbm>=2.1.0','requests', 'scikit-learn', 'scikit-image', 'gp_emulator>=1.6.0'], + url = 'https://github.com/MarcYin/SIAC', + license = "GNU Affero General Public License v3.0", + include_package_data = True, + packages = ['SIAC'], ) From 8a1d40ff9728883c108533ad114ccb7203108623 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 19 Sep 2018 16:28:31 +0100 Subject: [PATCH 005/103] tests --- .travis.yml | 3 ++- README.md | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f2471dd..b81579c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,9 @@ install: - conda config --set always_yes yes --set changeps1 no - conda update -q conda - conda info -a - - conda create -q -n test python=$TRAVIS_PYTHON_VERSION numpy numba scipy pytest pytest-cov coverage sphinx nose + - conda create -q -n test python=$TRAVIS_PYTHON_VERSION - source activate test + - conda install -c conda-forge gdal=2.2.4 - pip install -U codecov - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then pip install -U backports.functools_lru_cache; diff --git a/README.md b/README.md index ea8e284..ad4c2c2 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,9 @@ ### ucfafyi@ucl.ac.uk -[![PyPI version](https://badge.fury.io/py/SIAC.svg)](https://github.com/MarcYin/SIAC) +[![PyPI version](https://badge.fury.io/py/SIAC.svg)](https://pypi.org/project/SIAC/) +[![conda](https://anaconda.org/f0xy/siac/badges/version.svg)](https://anaconda.org/F0XY/siac) +[![conda update](https://anaconda.org/f0xy/siac/badges/latest_release_date.svg)](https://anaconda.org/F0XY/siac) This atmospheric correction method uses MODIS MCD43 BRDF product to get a coarse resolution simulation of earth surface. A model based on MODIS PSF is built to deal with the scale differences between MODIS and Sentinel 2 / Landsat 8. We uses the ECMWF CAMS prediction as a prior for the atmospheric states, coupling with 6S model to solve for the atmospheric parameters. We do not have topography correction and homogeneouse surface is used without considering the BRDF effects. From 8da56f14690fa30e26a487827c0d7cddd94857e0 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 19 Sep 2018 16:54:58 +0100 Subject: [PATCH 006/103] update --- README.md | 1 - SIAC/the_aerosol.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index ad4c2c2..4818d3d 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ [![PyPI version](https://badge.fury.io/py/SIAC.svg)](https://pypi.org/project/SIAC/) [![conda](https://anaconda.org/f0xy/siac/badges/version.svg)](https://anaconda.org/F0XY/siac) -[![conda update](https://anaconda.org/f0xy/siac/badges/latest_release_date.svg)](https://anaconda.org/F0XY/siac) This atmospheric correction method uses MODIS MCD43 BRDF product to get a coarse resolution simulation of earth surface. A model based on MODIS PSF is built to deal with the scale differences between MODIS and Sentinel 2 / Landsat 8. We uses the ECMWF CAMS prediction as a prior for the atmospheric states, coupling with 6S model to solve for the atmospheric parameters. We do not have topography correction and homogeneouse surface is used without considering the BRDF effects. diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index b7940b7..ac0648f 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -4,8 +4,6 @@ import os import gc import sys -import ogr -import osr import gdal import SIAC.kernels as kernels import logging @@ -18,6 +16,7 @@ import cPickle as pkl except: import pickle as pkl +from osgeo import ogr, osr from SIAC.smoothn import smoothn from functools import partial from SIAC.multi_process import parmap From d6bfd634cb13e6620b1ec3f0c87be1c741ed75ee Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 19 Sep 2018 17:15:57 +0100 Subject: [PATCH 007/103] test --- .travis.yml | 3 ++- README.md | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b81579c..294edd8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,12 +18,13 @@ install: - conda info -a - conda create -q -n test python=$TRAVIS_PYTHON_VERSION - source activate test + - conda update -c conda-forge --all - conda install -c conda-forge gdal=2.2.4 - pip install -U codecov - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then pip install -U backports.functools_lru_cache; fi - - python setup.py install + - pip install . script: - pytest diff --git a/README.md b/README.md index 4818d3d..2508b03 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,10 @@ ### ucfafyi@ucl.ac.uk -[![PyPI version](https://badge.fury.io/py/SIAC.svg)](https://pypi.org/project/SIAC/) -[![conda](https://anaconda.org/f0xy/siac/badges/version.svg)](https://anaconda.org/F0XY/siac) +[![PyPI version](https://img.shields.io/pypi/v/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) +[![conda](https://anaconda.org/f0xy/siac/badges/version.svg?longCache=true&style=flat)](https://anaconda.org/F0XY/siac) +[![py version](https://img.shields.io/pypi/pyversions/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) +[![build](https://img.shields.io/travis/SIAC/master.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) This atmospheric correction method uses MODIS MCD43 BRDF product to get a coarse resolution simulation of earth surface. A model based on MODIS PSF is built to deal with the scale differences between MODIS and Sentinel 2 / Landsat 8. We uses the ECMWF CAMS prediction as a prior for the atmospheric states, coupling with 6S model to solve for the atmospheric parameters. We do not have topography correction and homogeneouse surface is used without considering the BRDF effects. From ed528cc3b873a413164a9e7562112f999a00be3c Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 19 Sep 2018 17:25:57 +0100 Subject: [PATCH 008/103] tests.. --- README.md | 2 +- SIAC/the_aerosol.py | 4 ++-- SIAC/the_correction.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2508b03..7f013b9 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![PyPI version](https://img.shields.io/pypi/v/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) [![conda](https://anaconda.org/f0xy/siac/badges/version.svg?longCache=true&style=flat)](https://anaconda.org/F0XY/siac) [![py version](https://img.shields.io/pypi/pyversions/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) -[![build](https://img.shields.io/travis/SIAC/master.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) +[![build](https://img.shields.io/travis/MarcYin/SIAC/master.svg?longCache=true&style=flat)](https://travis-ci.org/MarcYin/SIAC/) This atmospheric correction method uses MODIS MCD43 BRDF product to get a coarse resolution simulation of earth surface. A model based on MODIS PSF is built to deal with the scale differences between MODIS and Sentinel 2 / Landsat 8. We uses the ECMWF CAMS prediction as a prior for the atmospheric states, coupling with 6S model to solve for the atmospheric parameters. We do not have topography correction and homogeneouse surface is used without considering the BRDF effects. diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index ac0648f..70481df 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -4,7 +4,7 @@ import os import gc import sys -import gdal +#import gdal import SIAC.kernels as kernels import logging import warnings @@ -16,7 +16,7 @@ import cPickle as pkl except: import pickle as pkl -from osgeo import ogr, osr +from osgeo import ogr, osr, gdal from SIAC.smoothn import smoothn from functools import partial from SIAC.multi_process import parmap diff --git a/SIAC/the_correction.py b/SIAC/the_correction.py index e9c64f6..d9fe279 100644 --- a/SIAC/the_correction.py +++ b/SIAC/the_correction.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import os import sys -import gdal +#import gdal import psutil import logging import warnings @@ -13,6 +13,7 @@ import cPickle as pkl except: import pickle as pkl +from osgeo import gdal from numpy import clip, uint8 from SIAC.multi_process import parmap from scipy.interpolate import griddata From f17678ecb0a35a1f11dfb3a6bf764de8dd8ae738 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 19 Sep 2018 17:56:57 +0100 Subject: [PATCH 009/103] test --- .setup.py.swp | Bin 0 -> 12288 bytes .travis.yml | 4 +--- 2 files changed, 1 insertion(+), 3 deletions(-) create mode 100644 .setup.py.swp diff --git a/.setup.py.swp b/.setup.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..d1c816587d52c1451a98d83f30be5d6c7d02bb5d GIT binary patch literal 12288 zcmeI2&u1tR1t&N5!C}-nl!+7h?;6`kP^EIm{$K&7(5M*R95e!jJ$?gWLjp(u2_OL^fCP{L5&sGObqB=lF~#s1P#rb1aU|8!xH z(52fyZ_`1Z3!~|6p@iWj-Si{yqIVn5y%pA7+7IVVxe#H&NN6;6uC{A&Tu3+eBbMqa z-ry#EAl0;NH`QKs6!*0%3bM}OTmpgWXlR5 z2D_+-qVgq&V&ZE_k6nfUhu~Sq%W@dvNppmGU|nRqoNn8pSNm)w#8bYiCm~XAYQpw2 z6Y`=_ZM~BvWN78%|9-$tqYJ~UN~(h1<*M+II$c|%n-e$IDvVax<%6tGV6-q5eL7yB zU74qsmfTvIOJPp9a{ zdmZc%MhP{7XVv_*916doiI!vG3lBmzRtPiWFb?q(1tB*X{;GfLC4myMcYjy%SO3uQt)n=JUbtYPE@(5lb u)wNKq=$D&=^^KdX%EFB`qo_@}rTH2mxzQCI3aaUy)}^Jgs literal 0 HcmV?d00001 diff --git a/.travis.yml b/.travis.yml index 294edd8..5950159 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,10 +16,8 @@ install: - conda config --set always_yes yes --set changeps1 no - conda update -q conda - conda info -a - - conda create -q -n test python=$TRAVIS_PYTHON_VERSION + - conda create -q -n test -c conda-forge python=$TRAVIS_PYTHON_VERSION gdal=2.2.4 scipy numpy psutil six lightgbm requests scikit-learn scikit-image gp_emulator - source activate test - - conda update -c conda-forge --all - - conda install -c conda-forge gdal=2.2.4 - pip install -U codecov - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then pip install -U backports.functools_lru_cache; From 16c9db2d0a9e907a9264969bce3984edac4c2997 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 19 Sep 2018 18:03:00 +0100 Subject: [PATCH 010/103] test --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5950159..1dc77d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,9 +16,10 @@ install: - conda config --set always_yes yes --set changeps1 no - conda update -q conda - conda info -a - - conda create -q -n test -c conda-forge python=$TRAVIS_PYTHON_VERSION gdal=2.2.4 scipy numpy psutil six lightgbm requests scikit-learn scikit-image gp_emulator + - conda create -q -n test -c conda-forge python=$TRAVIS_PYTHON_VERSION gdal=2.2.4 scipy numpy psutil six lightgbm requests scikit-learn scikit-image - source activate test - pip install -U codecov + - pip install gp_emulator - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then pip install -U backports.functools_lru_cache; fi From 0dc40fd85258e57efef84afa13e537405eeaad6f Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 19 Sep 2018 18:58:30 +0100 Subject: [PATCH 011/103] test --- .setup.py.swp | Bin 12288 -> 0 bytes MANIFEST.in | 2 +- {tests => SIAC/tests}/test_dependency.py | 0 {tests => SIAC/tests}/test_import.py | 0 setup.py | 2 +- 5 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 .setup.py.swp rename {tests => SIAC/tests}/test_dependency.py (100%) rename {tests => SIAC/tests}/test_import.py (100%) diff --git a/.setup.py.swp b/.setup.py.swp deleted file mode 100644 index d1c816587d52c1451a98d83f30be5d6c7d02bb5d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2&u1tR1t&N5!C}-nl!+7h?;6`kP^EIm{$K&7(5M*R95e!jJ$?gWLjp(u2_OL^fCP{L5&sGObqB=lF~#s1P#rb1aU|8!xH z(52fyZ_`1Z3!~|6p@iWj-Si{yqIVn5y%pA7+7IVVxe#H&NN6;6uC{A&Tu3+eBbMqa z-ry#EAl0;NH`QKs6!*0%3bM}OTmpgWXlR5 z2D_+-qVgq&V&ZE_k6nfUhu~Sq%W@dvNppmGU|nRqoNn8pSNm)w#8bYiCm~XAYQpw2 z6Y`=_ZM~BvWN78%|9-$tqYJ~UN~(h1<*M+II$c|%n-e$IDvVax<%6tGV6-q5eL7yB zU74qsmfTvIOJPp9a{ zdmZc%MhP{7XVv_*916doiI!vG3lBmzRtPiWFb?q(1tB*X{;GfLC4myMcYjy%SO3uQt)n=JUbtYPE@(5lb u)wNKq=$D&=^^KdX%EFB`qo_@}rTH2mxzQCI3aaUy)}^Jgs diff --git a/MANIFEST.in b/MANIFEST.in index 523cabd..96091c1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,4 +3,4 @@ include SIAC/util/* include SIAC/data/sen2cloud_detector.pkl include SIAC/data/.earthdata_auth include SIAC/spectral_mapping/* -recursive-include tests *.py +include SIAC/tests/*.py diff --git a/tests/test_dependency.py b/SIAC/tests/test_dependency.py similarity index 100% rename from tests/test_dependency.py rename to SIAC/tests/test_dependency.py diff --git a/tests/test_import.py b/SIAC/tests/test_import.py similarity index 100% rename from tests/test_import.py rename to SIAC/tests/test_import.py diff --git a/setup.py b/setup.py index 49ecb61..0ddfb21 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ readme = f.read().decode() setup(name ='SIAC', - version ='2.0.2', + version ='2.0.4', description = 'A sensor invariant Atmospheric Correction (SIAC)', long_description = readme, long_description_content_type ='text/markdown', From e5ad9bde3e8cd41e46693ae4d898740d314d5d8b Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 19 Sep 2018 19:44:18 +0100 Subject: [PATCH 012/103] test.... --- SIAC/tests/test_dependency.py | 14 +++++++------- SIAC/tests/test_import.py | 6 +++--- setup.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/SIAC/tests/test_dependency.py b/SIAC/tests/test_dependency.py index 36e7c67..65b869a 100644 --- a/SIAC/tests/test_dependency.py +++ b/SIAC/tests/test_dependency.py @@ -4,25 +4,25 @@ file_path = os.path.dirname(os.path.realpath(__file__)) def test_sen2cloud(): - assert os.path.exists(file_path.replace('tests', 'SIAC/data/sen2cloud_detector.pkl')) + assert os.path.exists(file_path.replace('tests', 'data/sen2cloud_detector.pkl')) def test_earthdata_auth(): - assert os.path.exists(file_path.replace('tests', 'SIAC/data/.earthdata_auth')) + assert os.path.exists(file_path.replace('tests', 'data/.earthdata_auth')) def test_spectral_mapping_1(): - assert os.path.exists(file_path.replace('tests', 'SIAC/spectral_mapping/TERRA_S2A_spectral_mapping.txt')) + assert os.path.exists(file_path.replace('tests', 'spectral_mapping/TERRA_S2A_spectral_mapping.txt')) def test_spectral_mapping_2(): - assert os.path.exists(file_path.replace('tests', 'SIAC/spectral_mapping/TERRA_S2B_spectral_mapping.txt')) + assert os.path.exists(file_path.replace('tests', 'spectral_mapping/TERRA_S2B_spectral_mapping.txt')) def test_spectral_mapping_3(): - assert os.path.exists(file_path.replace('tests', 'SIAC/spectral_mapping/TERRA_L8_spectral_mapping.txt')) + assert os.path.exists(file_path.replace('tests', 'spectral_mapping/TERRA_L8_spectral_mapping.txt')) def test_spectral_mapping_4(): - assert os.path.exists(file_path.replace('tests', 'SIAC/spectral_mapping/TERRA_TERRA_spectral_mapping.txt')) + assert os.path.exists(file_path.replace('tests', 'spectral_mapping/TERRA_TERRA_spectral_mapping.txt')) def test_l8_angle(): - assert os.path.exists(file_path.replace('tests', 'SIAC/util/l8_angles')) + assert os.path.exists(file_path.replace('tests', 'util/l8_angles')) if __name__ == '__main__': test_sen2cloud() diff --git a/SIAC/tests/test_import.py b/SIAC/tests/test_import.py index acb6439..afa8ade 100644 --- a/SIAC/tests/test_import.py +++ b/SIAC/tests/test_import.py @@ -1,9 +1,9 @@ import sys, os myPath = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, myPath + '/../') +sys.path.insert(0, myPath + '/../../') user_pass = 'user', 'pass' -with open(myPath.replace('tests', '') + '/SIAC/data/.earthdata_auth', 'wb') as f: +with open(myPath.replace('tests', '') + 'data/.earthdata_auth', 'wb') as f: for i in user_pass: f.write((i+'\n').encode()) @@ -11,5 +11,5 @@ from SIAC import SIAC_L8 -with open(myPath.replace('tests', '') + '/SIAC/data/.earthdata_auth', 'wb') as f: +with open(myPath.replace('tests', '') + 'data/.earthdata_auth', 'wb') as f: f.write(('').encode()) diff --git a/setup.py b/setup.py index 0ddfb21..66907c7 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ readme = f.read().decode() setup(name ='SIAC', - version ='2.0.4', + version ='2.0.7', description = 'A sensor invariant Atmospheric Correction (SIAC)', long_description = readme, long_description_content_type ='text/markdown', From eb99f35d0b208f9d1cd067c3fd6cdab1ddd00180 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 00:30:39 +0100 Subject: [PATCH 013/103] gdal --- SIAC/the_aerosol.py | 6 ++++-- SIAC/the_correction.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index 70481df..7a85de5 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -4,7 +4,9 @@ import os import gc import sys -#import gdal +import ogr +import osr +import gdal import SIAC.kernels as kernels import logging import warnings @@ -16,7 +18,7 @@ import cPickle as pkl except: import pickle as pkl -from osgeo import ogr, osr, gdal +#from osgeo import ogr, osr, gdal from SIAC.smoothn import smoothn from functools import partial from SIAC.multi_process import parmap diff --git a/SIAC/the_correction.py b/SIAC/the_correction.py index d9fe279..745df39 100644 --- a/SIAC/the_correction.py +++ b/SIAC/the_correction.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import os import sys -#import gdal +import gdal import psutil import logging import warnings @@ -13,7 +13,7 @@ import cPickle as pkl except: import pickle as pkl -from osgeo import gdal +#from osgeo import gdal from numpy import clip, uint8 from SIAC.multi_process import parmap from scipy.interpolate import griddata From 56f038f7009e7d3b3c3093a2cf63ae68965b9129 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 00:40:44 +0100 Subject: [PATCH 014/103] gdal --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 1dc77d5..b91c7ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ install: fi - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" + - sudo apt-get install -y gdal-bin - hash -r - conda config --set always_yes yes --set changeps1 no - conda update -q conda From c04df49c4292b49e38f509a02990388abfba1c04 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 01:01:35 +0100 Subject: [PATCH 015/103] gdal --- .travis.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index b91c7ec..ad4aa01 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,19 +12,14 @@ install: fi - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - - sudo apt-get install -y gdal-bin - hash -r - conda config --set always_yes yes --set changeps1 no - conda update -q conda + - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six lightgbm requests scikit-learn scikit-image gdal + - source activate test-environment - conda info -a - - conda create -q -n test -c conda-forge python=$TRAVIS_PYTHON_VERSION gdal=2.2.4 scipy numpy psutil six lightgbm requests scikit-learn scikit-image - - source activate test - - pip install -U codecov + - conda list - pip install gp_emulator - - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then - pip install -U backports.functools_lru_cache; - fi - - pip install . - + - conda install pip script: - pytest From 7889c14299b53db9ae863bb3255b0e39b8517fad Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 01:08:03 +0100 Subject: [PATCH 016/103] gdal --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ad4aa01..af4feac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ install: - export PATH="$HOME/miniconda/bin:$PATH" - hash -r - conda config --set always_yes yes --set changeps1 no + - conda config --add channels conda-forge - conda update -q conda - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six lightgbm requests scikit-learn scikit-image gdal - source activate test-environment From 443307acb6443c028d251aac6498e43ce827e0b1 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 10:26:24 +0100 Subject: [PATCH 017/103] coverage --- .travis.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index af4feac..1fd4f63 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,12 @@ install: - source activate test-environment - conda info -a - conda list - - pip install gp_emulator - conda install pip + - pip install gp_emulator + - pip install python-coveralls + - pip install codecov script: - - pytest + - py.test +after_success: + - codecov + - coveralls From 51176b9f89ba8b0acdb0c6022a37ee5243164d65 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 10:31:22 +0100 Subject: [PATCH 018/103] readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7f013b9..ad6b065 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![PyPI version](https://img.shields.io/pypi/v/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) +[![Lisence](https://img.shields.io/pypi/l/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) [![conda](https://anaconda.org/f0xy/siac/badges/version.svg?longCache=true&style=flat)](https://anaconda.org/F0XY/siac) [![py version](https://img.shields.io/pypi/pyversions/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) [![build](https://img.shields.io/travis/MarcYin/SIAC/master.svg?longCache=true&style=flat)](https://travis-ci.org/MarcYin/SIAC/) From 2932ad165a9aeae20525edd01912aa2598eb7532 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 10:50:12 +0100 Subject: [PATCH 019/103] coverage --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ad6b065..2231604 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,11 @@ [![PyPI version](https://img.shields.io/pypi/v/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) -[![Lisence](https://img.shields.io/pypi/l/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) [![conda](https://anaconda.org/f0xy/siac/badges/version.svg?longCache=true&style=flat)](https://anaconda.org/F0XY/siac) [![py version](https://img.shields.io/pypi/pyversions/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) +[![Coverage Status](https://coveralls.io/repos/github/MarcYin/SIAC/badge.svg?branch=master)](https://coveralls.io/github/MarcYin/SIAC?branch=master) [![build](https://img.shields.io/travis/MarcYin/SIAC/master.svg?longCache=true&style=flat)](https://travis-ci.org/MarcYin/SIAC/) +[![Lisence](https://img.shields.io/pypi/l/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) This atmospheric correction method uses MODIS MCD43 BRDF product to get a coarse resolution simulation of earth surface. A model based on MODIS PSF is built to deal with the scale differences between MODIS and Sentinel 2 / Landsat 8. We uses the ECMWF CAMS prediction as a prior for the atmospheric states, coupling with 6S model to solve for the atmospheric parameters. We do not have topography correction and homogeneouse surface is used without considering the BRDF effects. From 7c57c26ef402c7a660be4b02a8b09a630a2c0dc4 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 11:25:52 +0100 Subject: [PATCH 020/103] travis --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1fd4f63..1b4b33e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,10 +22,11 @@ install: - conda list - conda install pip - pip install gp_emulator + - pip install pytest-cov - pip install python-coveralls - pip install codecov script: - - py.test + - py.test --cov=tests after_success: - codecov - coveralls From b6595175d223ba62ba437fcf18776641c33c9f95 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 11:38:21 +0100 Subject: [PATCH 021/103] coverage --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1b4b33e..9ca2cf9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,14 +16,12 @@ install: - conda config --set always_yes yes --set changeps1 no - conda config --add channels conda-forge - conda update -q conda - - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six lightgbm requests scikit-learn scikit-image gdal + - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six lightgbm requests scikit-learn scikit-image gdal coveralls pytest-cov python-coveralls - source activate test-environment - conda info -a - conda list - conda install pip - pip install gp_emulator - - pip install pytest-cov - - pip install python-coveralls - pip install codecov script: - py.test --cov=tests From c1718dbd5c27a20ddbf4c41001f84199e8d8a405 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 12:31:48 +0100 Subject: [PATCH 022/103] coverage --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 9ca2cf9..8805469 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ install: - conda install pip - pip install gp_emulator - pip install codecov + - pip install coverage>=4.4 script: - py.test --cov=tests after_success: From 5eb07c6e825867f48c8ebe1ffac61a3fda859017 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 13:07:41 +0100 Subject: [PATCH 023/103] coverage --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8805469..6f66f3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,14 +16,13 @@ install: - conda config --set always_yes yes --set changeps1 no - conda config --add channels conda-forge - conda update -q conda - - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six lightgbm requests scikit-learn scikit-image gdal coveralls pytest-cov python-coveralls + - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six lightgbm requests scikit-learn scikit-image gdal coveralls pytest-cov python-coveralls coverage=4.5.1 - source activate test-environment - conda info -a - conda list - conda install pip - pip install gp_emulator - pip install codecov - - pip install coverage>=4.4 script: - py.test --cov=tests after_success: From 64d016104a0346a1a4e76ff52c205e93738af527 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 13:15:19 +0100 Subject: [PATCH 024/103] coverage --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6f66f3d..2df9e77 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,12 +16,13 @@ install: - conda config --set always_yes yes --set changeps1 no - conda config --add channels conda-forge - conda update -q conda - - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six lightgbm requests scikit-learn scikit-image gdal coveralls pytest-cov python-coveralls coverage=4.5.1 + - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six lightgbm requests scikit-learn scikit-image gdal coveralls pytest-cov coverage=4.5.1 - source activate test-environment - conda info -a - conda list - conda install pip - pip install gp_emulator + - pip install coveralls - pip install codecov script: - py.test --cov=tests From 86cb8362e9aa0b3f6721587e2577e2750f2a454a Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 14:46:31 +0100 Subject: [PATCH 025/103] coverage --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2df9e77..dcaf1f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,8 @@ install: - pip install coveralls - pip install codecov script: - - py.test --cov=tests + - py.test + - coverage xml after_success: - codecov - coveralls From 8965841db8c5ae06b7b5256df95b9fdf57640fda Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 15:00:48 +0100 Subject: [PATCH 026/103] codedev --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2231604..4ae7543 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,9 @@ [![PyPI version](https://img.shields.io/pypi/v/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) [![conda](https://anaconda.org/f0xy/siac/badges/version.svg?longCache=true&style=flat)](https://anaconda.org/F0XY/siac) [![py version](https://img.shields.io/pypi/pyversions/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) -[![Coverage Status](https://coveralls.io/repos/github/MarcYin/SIAC/badge.svg?branch=master)](https://coveralls.io/github/MarcYin/SIAC?branch=master) [![build](https://img.shields.io/travis/MarcYin/SIAC/master.svg?longCache=true&style=flat)](https://travis-ci.org/MarcYin/SIAC/) +[![codecov](https://codecov.io/gh/MarcYin/SIAC/branch/master/graph/badge.svg?longCache=true&style=flat)](https://codecov.io/gh/MarcYin/SIAC) +[![Coverage Status](https://coveralls.io/repos/github/MarcYin/SIAC/badge.svg?branch=master)](https://coveralls.io/github/MarcYin/SIAC?branch=master) [![Lisence](https://img.shields.io/pypi/l/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) This atmospheric correction method uses MODIS MCD43 BRDF product to get a coarse resolution simulation of earth surface. A model based on MODIS PSF is built to deal with the scale differences between MODIS and Sentinel 2 / Landsat 8. We uses the ECMWF CAMS prediction as a prior for the atmospheric states, coupling with 6S model to solve for the atmospheric parameters. We do not have topography correction and homogeneouse surface is used without considering the BRDF effects. From c5f5e4cdc00d4f0dab2ef73dedddb42cc5f98b7c Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 16:00:04 +0100 Subject: [PATCH 027/103] compatible with gdal 2.3 --- .travis.yml | 2 +- SIAC/reproject.py | 2 +- SIAC/the_aerosol.py | 16 ++++++++-------- SIAC/the_correction.py | 24 ++++++++++++------------ 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index dcaf1f0..7ea5694 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ install: - pip install coveralls - pip install codecov script: - - py.test + - py.test --cov - coverage xml after_success: - codecov diff --git a/SIAC/reproject.py b/SIAC/reproject.py index 5537313..f2067bc 100644 --- a/SIAC/reproject.py +++ b/SIAC/reproject.py @@ -26,7 +26,7 @@ def __init__(self, source_img, yRes = None, xSize = None, ySize = None, - resample = gdal.GRIORA_Bilinear + resample = 1 ): self.source_img = source_img diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index 7a85de5..c5894f3 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -206,7 +206,7 @@ def _mask_bad_pix(self): ySize=self.full_res[0], \ srcNodata = np.nan,\ outputType= gdal.GDT_Float32,\ - resample = gdal.GRIORA_NearestNeighbour).data + resample = 0).data cloud = cloud.astype(bool) RED = None BLUE = None @@ -262,7 +262,7 @@ def _create_band_gs(self,): self._toa_bands = [] for band in self.toa_bands: g = gdal.Warp('', band, format = 'MEM', xRes = self.pixel_res, yRes = self.pixel_res, warpOptions = ['NUM_THREADS=ALL_CPUS'],\ - srcNodata = 0, dstNodata=0, cutlineDSName= self.aoi, cropToCutline=True, resampleAlg = gdal.GRIORA_NearestNeighbour) + srcNodata = 0, dstNodata=0, cutlineDSName= self.aoi, cropToCutline=True, resampleAlg = 0) self._toa_bands.append(g) self.example_file = self._toa_bands[0] x_max_pix = self.example_file.RasterXSize * self.pixel_res @@ -282,7 +282,7 @@ def _resamplers(self,): yRes=self.aero_res*0.5, \ srcNodata = None,\ outputType= gdal.GDT_Float32, \ - resample = gdal.GRIORA_NearestNeighbour).g + resample = 0 ).g # GRIORA_NearestNeighbour due to version changes... self.bilinear_resampler = lambda fname: reproject_data(fname, \ self.example_file, \ @@ -290,7 +290,7 @@ def _resamplers(self,): yRes=self.aero_res*0.5, \ srcNodata = 0,\ outputType= gdal.GDT_Float32,\ - resample = gdal.GRIORA_Bilinear).g + resample = 1 ).g # GRIORA_Bilinear def _var_parser(self, var): if os.path.exists(str(var)): @@ -412,7 +412,7 @@ def _get_bounds(self,): def _read_MCD43(self,fnames): def warp_data(fname, aoi, xRes, yRes): g = gdal.Warp('',fname, format = 'MEM', dstNodata=0, cutlineDSName=aoi, xRes = \ - xRes, yRes = yRes, cropToCutline=True, resampleAlg = gdal.GRIORA_NearestNeighbour) + xRes, yRes = yRes, cropToCutline=True, resampleAlg = 0) return g.ReadAsArray() par = partial(warp_data, aoi = self.aoi, xRes = self.aero_res*0.5, yRes = self.aero_res*0.5) n_files = int(len(fnames)/2) @@ -448,7 +448,7 @@ def _get_boa(self, temporal_filling = 16): fnames.append(fname) das, ws = self._read_MCD43(fnames) mg = gdal.Warp('',fnames[0], format = 'MEM', dstNodata= None, xRes = self.aero_res*0.5, yRes = \ - self.aero_res*0.5, cutlineDSName=self.aoi, cropToCutline=True, resampleAlg = gdal.GRIORA_NearestNeighbour) + self.aero_res*0.5, cutlineDSName=self.aoi, cropToCutline=True, resampleAlg = 0) hg = self.example_file self.hx, self.hy, hmask, rmask = self._get_index(mg, hg) @@ -500,11 +500,11 @@ def _annoying_angles(self, destination): _view_angles = [] for fname in self._sun_angles: ang = reproject_data(fname, mg, srcNodata = None, resample = \ - gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data + 0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data _sun_angles.append(ang) for fname in self._view_angles: ang = reproject_data(fname, mg, srcNodata = None, resample = \ - gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data + 0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data _view_angles.append(ang) _view_angles = np.squeeze(np.array(_view_angles)) _sun_angles = np.squeeze(np.array(_sun_angles)) diff --git a/SIAC/the_correction.py b/SIAC/the_correction.py index 745df39..50ecdf4 100644 --- a/SIAC/the_correction.py +++ b/SIAC/the_correction.py @@ -154,7 +154,7 @@ def _create_band_gs(self,): self._toa_bands = [] for band in self.toa_bands: g = gdal.Warp('', band, format = 'MEM', srcNodata = 0, dstNodata=0, warpOptions = \ - ['NUM_THREADS=ALL_CPUS'], cutlineDSName= self.aoi, cropToCutline=True, resampleAlg = gdal.GRIORA_NearestNeighbour) + ['NUM_THREADS=ALL_CPUS'], cutlineDSName= self.aoi, cropToCutline=True, resampleAlg = 0) self._toa_bands.append(g) self.example_file = self._toa_bands[0] if self.cloud_mask is None: @@ -163,7 +163,7 @@ def _create_band_gs(self,): self.mask_g = array_to_raster(self.mask.astype(float), self.example_file) self.mask = self.mask.astype(bool) self.mg = gdal.Warp('', band, format = 'MEM', srcNodata = None, xRes = self.block_size, yRes = \ - self.block_size, cutlineDSName= self.aoi, cropToCutline=True, resampleAlg = gdal.GRIORA_NearestNeighbour) + self.block_size, cutlineDSName= self.aoi, cropToCutline=True, resampleAlg = 0) def _load_xa_xb_xc_emus(self,): xap_emu = glob(self.emus_dir + '/isotropic_%s_emulators_correction_xap_%s.pkl'%(self.sensor, self.satellite))[0] @@ -216,11 +216,11 @@ def _annoying_angles(self,): _view_angles = [] for fname in self._sun_angles: ang = reproject_data(fname, self.mg, srcNodata = None, resample = \ - gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data + 0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data _sun_angles.append(ang) for fname in self._view_angles: ang = reproject_data(fname, self.mg, srcNodata = None, resample = \ - gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data + 0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data _view_angles.append(ang) _view_angles = np.array(_view_angles) _sun_angles = np.squeeze(np.array(_sun_angles)) @@ -266,7 +266,7 @@ def _parse_atmo(self,): datetime.strftime(self.obs_time, '%Y_%m_%d')+'_%s.tif'%cams_names[i]]) var_g = self._var_parser(atmos[i]) _g = reproject_data(var_g, self.mg, srcNodata = None, resample = \ - gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).g + 0, dstNodata=np.nan, outputType= gdal.GDT_Float32).g offset = var_g.GetOffset() scale = var_g.GetScale() data = _g.GetRasterBand(int(time_ind+1)).ReadAsArray() * scale + offset @@ -277,11 +277,11 @@ def _parse_atmo(self,): for i in range(3): var_g = self._var_parser(atmos[i]) data = reproject_data(var_g, self.mg, srcNodata = np.nan, resample = \ - gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data + 0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data atmos[i] = data * self.atmo_scale[i] unc_g = self._var_parser(atmo_uncs[i]) ug = reproject_data(unc_g, self.mg, srcNodata = np.nan, resample = \ - gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data + 0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data atmo_uncs[i] = ug self._aot, self._tcwv, self._tco3 = atmos @@ -293,7 +293,7 @@ def _parse_aux(self,): for i in range(len(auxs)): var_g = self._var_parser(auxs[i]) dat = reproject_data(var_g, self.mg, srcNodata = 0, resample = \ - gdal.GRIORA_NearestNeighbour, dstNodata=np.nan, outputType= gdal.GDT_Float32).data + 0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data auxs[i] = dat * scales[i] self._ele, self._cmask = auxs self._ele[np.isnan(self._ele)] = 0 @@ -406,12 +406,12 @@ def _do_chunk(self, ind): x_off = x_end - x_start toa = toa_g.ReadAsArray(x_start, 0, x_off, int(y_size)) - _g = gdal.Warp('', toa_g, format = 'MEM', outputBounds = [xmin, geo[3] + y_size * geo[5], xmax, geo[3]], resampleAlg = gdal.GRIORA_NearestNeighbour) + _g = gdal.Warp('', toa_g, format = 'MEM', outputBounds = [xmin, geo[3] + y_size * geo[5], xmax, geo[3]], resampleAlg = 0) xRes = yRes = int(geo[1]) xps = [self.xap_H[ind], self.xbp_H[ind], self.xcp_H[ind]] for j in range(3): xps[j] = reproject_data(xps[j], _g,xRes=xRes, yRes = yRes,xSize=x_off, \ - ySize=y_size, resample = gdal.GRIORA_NearestNeighbour, outputType= gdal.GDT_Float32).data + ySize=y_size, resample = 0, outputType= gdal.GDT_Float32).data xap_H, xbp_H, xcp_H = xps toa = toa * self.ref_scale + self.ref_off @@ -422,7 +422,7 @@ def _do_chunk(self, ind): for j in range(3): #var_g = xhs[j] xhs[j] = reproject_data(xhs[j], _g, xRes=xRes, yRes = yRes, xSize=x_off, \ - ySize=y_size, resample = gdal.GRIORA_NearestNeighbour, \ + ySize=y_size, resample = 0, \ outputType= gdal.GDT_Float32).data.transpose(1,2,0) xap_dH, xbp_dH, xcp_dH = xhs @@ -443,7 +443,7 @@ def _do_chunk(self, ind): tmp = [] for unc_g in [self._aot_unc, self._tcwv_unc, self._tco3_unc]: - unc = reproject_data(unc_g, _g, xSize=x_off, ySize=y_size, resample = gdal.GRIORA_NearestNeighbour, outputType= gdal.GDT_Float32).data + unc = reproject_data(unc_g, _g, xSize=x_off, ySize=y_size, resample = 0, outputType= gdal.GDT_Float32).data tmp.append(unc) aot_unc, tcwv_unc, tco3_unc = tmp; del tmp unc = np.sqrt(aot_dH ** 2 * aot_unc**2 + tcwv_dH ** 2 * tcwv_unc**2 + tco3_dH ** 2 * tco3_unc**2 + toa_dH**2 * 0.015**2) From 0ec8edce6f3543e5caabe1dd601789c82e16fae1 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 17:08:59 +0100 Subject: [PATCH 028/103] auto conda --- .travis.yml | 3 ++- conda_upload.sh | 11 ++++++++ meta.yaml | 71 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 conda_upload.sh create mode 100644 meta.yaml diff --git a/.travis.yml b/.travis.yml index 7ea5694..6f5ccd9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ install: - conda info -a - conda list - conda install pip - - pip install gp_emulator + - pip install gp_emulator=1.6.4 - pip install coveralls - pip install codecov script: @@ -30,3 +30,4 @@ script: after_success: - codecov - coveralls + - test $TRAVIS_BRANCH = "master" && conda install conda-build && conda install anaconda-client && bash conda_upload.sh diff --git a/conda_upload.sh b/conda_upload.sh new file mode 100644 index 0000000..9280188 --- /dev/null +++ b/conda_upload.sh @@ -0,0 +1,11 @@ +# Only need to change these two variables +PKG_NAME=siac +USER=f0xy + +OS=$TRAVIS_OS_NAME-64 +mkdir ~/conda-bld +conda config --set anaconda_upload no +export CONDA_BLD_PATH=~/conda-bld +export VERSION=`date +%Y.%m.%d` +conda build . -c conda-forge -c uvcdat +anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $CONDA_BLD_PATH/$OS/$PKG_NAME-$VERSION-0.tar.bz2 --force diff --git a/meta.yaml b/meta.yaml new file mode 100644 index 0000000..c0a2066 --- /dev/null +++ b/meta.yaml @@ -0,0 +1,71 @@ +package: + name: siac + version: {{ environ['VERSION'] }} + +source: + url: https://github.com/MarcYin/SIAC.git + + +build: + script: python setup.py install + script_env: + - VERSION + - CONDA_BLD_PATH + +requirements: + host: + - python + - setuptools + - gdal >=2.1,<2.3 + - numpy >=1.13 + - scipy >=1.0 + - psutil + - six + - lightgbm >=2.1.0 + - requests + - scikit-learn + - scikit-image + run: + - python + - gdal >=2.1,<2.3 + - numpy >=1.13 + - scipy >=1.0 + - psutil + - six + - lightgbm >=2.1.0 + - requests + - scikit-learn + - scikit-image + +test: + requires: + - pytest + commands: + - pytest .. + +about: + home: https://github.com/MarcYin/SIAC + license: GNU Affero General Public License v3.0 + license_family: AGPL + license_file: '' + summary: A sensor invariant Atmospheric Correction (SIAC) + description: "# A sensor invariant Atmospheric Correction (SIAC)\n### Feng Yin\n### Department of Geography, UCL\n### ucfafyi@ucl.ac.uk\n\n\n[![PyPI version](https://img.shields.io/pypi/v/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/)\n\ + [![conda](https://anaconda.org/f0xy/siac/badges/version.svg?longCache=true&style=flat)](https://anaconda.org/F0XY/siac)\n[![py version](https://img.shields.io/pypi/pyversions/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/)\n\ + [![build](https://img.shields.io/travis/MarcYin/SIAC/master.svg?longCache=true&style=flat)](https://travis-ci.org/MarcYin/SIAC/)\n\nThis atmospheric correction method uses MODIS MCD43 BRDF product to\ + \ get a coarse resolution simulation of earth surface. A model based on MODIS PSF is built to deal with the scale differences between MODIS and Sentinel 2 / Landsat 8. We uses the ECMWF CAMS prediction\ + \ as a prior for the atmospheric states, coupling with 6S model to solve for the atmospheric parameters. We do not have topography correction and homogeneouse surface is used without considering the\ + \ BRDF effects.\n\n\n## Data needed:\n* MCD43 : 16 days before and 16 days after the Sentinel 2 / Landsat 8 sensing date\n* ECMWF CAMS Near Real Time prediction: a time step of 3 hours with the start\ + \ time of 00:00:00 over the date, and data from 01/04/2015 are mirrored in UCL server at: http://www2.geog.ucl.ac.uk/~ucfafyi/cams/\n* Global DEM: Global DEM VRT file built from ASTGTM2 DEM, and most\ + \ of the DEM over land are mirrored in UCL server at: http://www2.geog.ucl.ac.uk/~ucfafyi/eles/\n* Emulators: emulators for atmospheric path reflectance, total transmitance and single scattering Albedo,\ + \ and the emulators for Sentinel 2, Landsat 8 and MODIS trained with 6S.V2 can be found at: http://www2.geog.ucl.ac.uk/~ucfafyi/emus/\n\n## Installation:\n\nDownload this repositories either by `git\ + \ clone git@ github.com:MarcYin/SIAC.git` or download the zip file and unzip it. In the main directory of it excute `pip install .` , or `pip install SIAC` or `conda install -c f0xy siac`\n\nTo save\ + \ your time for installing GDAL and lightgbm, please use:\n\n```bash\nconda install -c conda-forge gdal=2.2.4\nconda install -c conda-forge lightgbm\n```\n\n\n\nThe typical usage for Sentinel 2 and\ + \ Landsat 8:\n```python\nfrom SIAC import SIAC_S2\nSIAC_S2('/directory/where/you/store/S2/data/') # this can be either from AWS or Senitinel offical package\n```\n```python\nfrom SIAC import SIAC_L8\ + \ \nSIAC_L8('/directory/where/you/store/L8/data/') \n``` \n\nAn example of correction for Landsat 5 for a more detailed demostration\ + \ of the usage is shown [here](https://github.com/MarcYin/Global-analysis-ready-dataset)\n\n## Examples and Map:\n\nA [page](http://www2.geog.ucl.ac.uk/~ucfafyi/Atmo_Cor/index.html) shows some correction\ + \ samples.\n\nA [map](http://www2.geog.ucl.ac.uk/~ucfafyi/map) for comparison between TOA and BOA.\n\n### LICENSE\nGNU GENERAL PUBLIC LICENSE V3\n\n\n" + doc_url: '' + dev_url: '' + +extra: + recipe-maintainers: '' From 74f9644afd563cab6b812ecb11adb691bc10ddbb Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 17:22:09 +0100 Subject: [PATCH 029/103] emulators --- .travis.yml | 2 +- conda_upload.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6f5ccd9..78b0151 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ install: - conda info -a - conda list - conda install pip - - pip install gp_emulator=1.6.4 + - pip install gp_emulator==1.6.4 - pip install coveralls - pip install codecov script: diff --git a/conda_upload.sh b/conda_upload.sh index 9280188..55d9872 100644 --- a/conda_upload.sh +++ b/conda_upload.sh @@ -7,5 +7,5 @@ mkdir ~/conda-bld conda config --set anaconda_upload no export CONDA_BLD_PATH=~/conda-bld export VERSION=`date +%Y.%m.%d` -conda build . -c conda-forge -c uvcdat +conda build . -c conda-forge anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $CONDA_BLD_PATH/$OS/$PKG_NAME-$VERSION-0.tar.bz2 --force From f2ed3d461732cda21bfada5a274eab543ef70c42 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 17:38:21 +0100 Subject: [PATCH 030/103] emulator.. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 78b0151..455d1c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ install: - conda info -a - conda list - conda install pip - - pip install gp_emulator==1.6.4 + - pip install gp_emulator - pip install coveralls - pip install codecov script: From 365b54ffd612b46e5533d36112a3f0d039391536 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 18:37:36 +0100 Subject: [PATCH 031/103] auto conda --- .travis.yml | 2 +- conda_upload.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 455d1c7..8c629cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ install: - conda info -a - conda list - conda install pip - - pip install gp_emulator + - pip install gp_emulator==1.6.1 - pip install coveralls - pip install codecov script: diff --git a/conda_upload.sh b/conda_upload.sh index 55d9872..59935d8 100644 --- a/conda_upload.sh +++ b/conda_upload.sh @@ -8,4 +8,6 @@ conda config --set anaconda_upload no export CONDA_BLD_PATH=~/conda-bld export VERSION=`date +%Y.%m.%d` conda build . -c conda-forge +ls $CONDA_BLD_PATH +ls $CONDA_BLD_PATH/$OS anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $CONDA_BLD_PATH/$OS/$PKG_NAME-$VERSION-0.tar.bz2 --force From e493f7877c45b41fec52571042061394c06a5187 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 18:58:55 +0100 Subject: [PATCH 032/103] auto conda .... --- meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta.yaml b/meta.yaml index c0a2066..4f7c840 100644 --- a/meta.yaml +++ b/meta.yaml @@ -3,8 +3,8 @@ package: version: {{ environ['VERSION'] }} source: - url: https://github.com/MarcYin/SIAC.git - + git_url: https://github.com/MarcYin/SIAC.git + git_tag: master build: script: python setup.py install From 06dc72966624c5ac255a2255149b93e2be66fc23 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 20 Sep 2018 22:36:51 +0100 Subject: [PATCH 033/103] auto conda --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8c629cc..432839f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,4 +30,4 @@ script: after_success: - codecov - coveralls - - test $TRAVIS_BRANCH = "master" && conda install conda-build && conda install anaconda-client && bash conda_upload.sh + - test $TRAVIS_BRANCH = "master" && conda install conda-build && conda install anaconda-client && PKG_NAME=siac && USER=f0xy && OS=$TRAVIS_OS_NAME-64 && mkdir ~/conda-bld && conda config --set anaconda_upload no && export CONDA_BLD_PATH=~/conda-bld && export VERSION=`date +%Y.%m.%d` && conda build . -c conda-forge && ls -lah $CONDA_BLD_PATH/$OS && anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $CONDA_BLD_PATH/$OS/$PKG_NAME-$VERSION-0.tar.bz2 --force From 91801b10d68035590ac47351fb9b55c50221ee3a Mon Sep 17 00:00:00 2001 From: MarcYin Date: Fri, 21 Sep 2018 01:03:04 +0100 Subject: [PATCH 034/103] seems work... --- meta.yaml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/meta.yaml b/meta.yaml index 4f7c840..fd9c8fe 100644 --- a/meta.yaml +++ b/meta.yaml @@ -7,7 +7,7 @@ source: git_tag: master build: - script: python setup.py install + script: pip install gp_emulator==1.6.1 && python setup.py install script_env: - VERSION - CONDA_BLD_PATH diff --git a/setup.py b/setup.py index 66907c7..8b39f38 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.6'], install_requires = ['gdal>=2.1, <2.3', 'numpy>=1.13', 'scipy>=1.0', 'psutil','six', - 'lightgbm>=2.1.0','requests', 'scikit-learn', 'scikit-image', 'gp_emulator>=1.6.0'], + 'lightgbm>=2.1.0','requests', 'scikit-learn', 'scikit-image', 'gp_emulator==1.6.1'], url = 'https://github.com/MarcYin/SIAC', license = "GNU Affero General Public License v3.0", include_package_data = True, From 4118914212c2fb8d03f10dc10f4b222f1af2b00f Mon Sep 17 00:00:00 2001 From: MarcYin Date: Fri, 21 Sep 2018 09:58:29 +0100 Subject: [PATCH 035/103] auto conda! --- .travis.yml | 2 +- conda_upload.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 432839f..55699e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,4 +30,4 @@ script: after_success: - codecov - coveralls - - test $TRAVIS_BRANCH = "master" && conda install conda-build && conda install anaconda-client && PKG_NAME=siac && USER=f0xy && OS=$TRAVIS_OS_NAME-64 && mkdir ~/conda-bld && conda config --set anaconda_upload no && export CONDA_BLD_PATH=~/conda-bld && export VERSION=`date +%Y.%m.%d` && conda build . -c conda-forge && ls -lah $CONDA_BLD_PATH/$OS && anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $CONDA_BLD_PATH/$OS/$PKG_NAME-$VERSION-0.tar.bz2 --force + - test $TRAVIS_BRANCH = "master" && conda install conda-build && conda install anaconda-client && PKG_NAME=siac && USER=f0xy && OS=$TRAVIS_OS_NAME-64 && mkdir ~/conda-bld && conda config --set anaconda_upload no && export CONDA_BLD_PATH=~/conda-bld && export VERSION=`date +%Y.%m.%d` && conda build . -c conda-forge && ls -lah $CONDA_BLD_PATH/$OS && anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $(ls $CONDA_BLD_PATH/$OS/$PKG_NAME-$VERSION*.tar.bz2) --force diff --git a/conda_upload.sh b/conda_upload.sh index 59935d8..837e6a1 100644 --- a/conda_upload.sh +++ b/conda_upload.sh @@ -10,4 +10,4 @@ export VERSION=`date +%Y.%m.%d` conda build . -c conda-forge ls $CONDA_BLD_PATH ls $CONDA_BLD_PATH/$OS -anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $CONDA_BLD_PATH/$OS/$PKG_NAME-$VERSION-0.tar.bz2 --force +anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $(ls $CONDA_BLD_PATH/$OS/$PKG_NAME-$VERSION*.tar.bz2) --force From 970eabd534380bd1ecd30eb9e7903ca272f7cac0 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Fri, 21 Sep 2018 10:32:26 +0100 Subject: [PATCH 036/103] update conda description --- meta.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/meta.yaml b/meta.yaml index fd9c8fe..ccbef53 100644 --- a/meta.yaml +++ b/meta.yaml @@ -49,9 +49,15 @@ about: license_family: AGPL license_file: '' summary: A sensor invariant Atmospheric Correction (SIAC) - description: "# A sensor invariant Atmospheric Correction (SIAC)\n### Feng Yin\n### Department of Geography, UCL\n### ucfafyi@ucl.ac.uk\n\n\n[![PyPI version](https://img.shields.io/pypi/v/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/)\n\ - [![conda](https://anaconda.org/f0xy/siac/badges/version.svg?longCache=true&style=flat)](https://anaconda.org/F0XY/siac)\n[![py version](https://img.shields.io/pypi/pyversions/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/)\n\ - [![build](https://img.shields.io/travis/MarcYin/SIAC/master.svg?longCache=true&style=flat)](https://travis-ci.org/MarcYin/SIAC/)\n\nThis atmospheric correction method uses MODIS MCD43 BRDF product to\ + description: "# A sensor invariant Atmospheric Correction (SIAC)\n### Feng Yin\n### Department of Geography, UCL\n### ucfafyi@ucl.ac.uk\n\n\n\ + [![PyPI version](https://img.shields.io/pypi/v/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/)\n\ + [![conda](https://anaconda.org/f0xy/siac/badges/version.svg?longCache=true&style=flat)](https://anaconda.org/F0XY/siac)\n\ + [![py version](https://img.shields.io/pypi/pyversions/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/)\n\ + [![py version](https://img.shields.io/pypi/pyversions/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/)\n\ + [![build](https://img.shields.io/travis/MarcYin/SIAC/master.svg?longCache=true&style=flat)](https://travis-ci.org/MarcYin/SIAC/)\n\ + [![codecov](https://codecov.io/gh/MarcYin/SIAC/branch/master/graph/badge.svg?longCache=true&style=flat)](https://codecov.io/gh/MarcYin/SIAC)\n\ + [![Lisence](https://img.shields.io/pypi/l/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/)\n\ + \nThis atmospheric correction method uses MODIS MCD43 BRDF product to\ \ get a coarse resolution simulation of earth surface. A model based on MODIS PSF is built to deal with the scale differences between MODIS and Sentinel 2 / Landsat 8. We uses the ECMWF CAMS prediction\ \ as a prior for the atmospheric states, coupling with 6S model to solve for the atmospheric parameters. We do not have topography correction and homogeneouse surface is used without considering the\ \ BRDF effects.\n\n\n## Data needed:\n* MCD43 : 16 days before and 16 days after the Sentinel 2 / Landsat 8 sensing date\n* ECMWF CAMS Near Real Time prediction: a time step of 3 hours with the start\ @@ -60,8 +66,8 @@ about: \ and the emulators for Sentinel 2, Landsat 8 and MODIS trained with 6S.V2 can be found at: http://www2.geog.ucl.ac.uk/~ucfafyi/emus/\n\n## Installation:\n\nDownload this repositories either by `git\ \ clone git@ github.com:MarcYin/SIAC.git` or download the zip file and unzip it. In the main directory of it excute `pip install .` , or `pip install SIAC` or `conda install -c f0xy siac`\n\nTo save\ \ your time for installing GDAL and lightgbm, please use:\n\n```bash\nconda install -c conda-forge gdal=2.2.4\nconda install -c conda-forge lightgbm\n```\n\n\n\nThe typical usage for Sentinel 2 and\ - \ Landsat 8:\n```python\nfrom SIAC import SIAC_S2\nSIAC_S2('/directory/where/you/store/S2/data/') # this can be either from AWS or Senitinel offical package\n```\n```python\nfrom SIAC import SIAC_L8\ - \ \nSIAC_L8('/directory/where/you/store/L8/data/') \n``` \n\nAn example of correction for Landsat 5 for a more detailed demostration\ + \ Landsat 8:\n\n```python\nfrom SIAC import SIAC_S2\nSIAC_S2('/directory/where/you/store/S2/data/') # this can be either from AWS or Senitinel offical package\n```\n\n```python\nfrom SIAC import SIAC_L8\ + \\nSIAC_L8('/directory/where/you/store/L8/data/') \n``` \n\nAn example of correction for Landsat 5 for a more detailed demostration\ \ of the usage is shown [here](https://github.com/MarcYin/Global-analysis-ready-dataset)\n\n## Examples and Map:\n\nA [page](http://www2.geog.ucl.ac.uk/~ucfafyi/Atmo_Cor/index.html) shows some correction\ \ samples.\n\nA [map](http://www2.geog.ucl.ac.uk/~ucfafyi/map) for comparison between TOA and BOA.\n\n### LICENSE\nGNU GENERAL PUBLIC LICENSE V3\n\n\n" doc_url: '' From 897da0ba55f3acea62f74e4c0980154f3539c70b Mon Sep 17 00:00:00 2001 From: MarcYin Date: Fri, 21 Sep 2018 11:48:39 +0100 Subject: [PATCH 037/103] add conda readme --- conda_readme.md | 58 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 conda_readme.md diff --git a/conda_readme.md b/conda_readme.md new file mode 100644 index 0000000..684727e --- /dev/null +++ b/conda_readme.md @@ -0,0 +1,58 @@ +# A sensor invariant Atmospheric Correction (SIAC) +### Feng Yin +### Department of Geography, UCL +### ucfafyi@ucl.ac.uk + + +[![PyPI version](https://img.shields.io/pypi/v/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) +[![conda](https://anaconda.org/f0xy/siac/badges/version.svg?longCache=true&style=flat)](https://anaconda.org/F0XY/siac) +[![py version](https://img.shields.io/pypi/pyversions/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) +[![build](https://img.shields.io/travis/MarcYin/SIAC/master.svg?longCache=true&style=flat)](https://travis-ci.org/MarcYin/SIAC/) +[![codecov](https://codecov.io/gh/MarcYin/SIAC/branch/master/graph/badge.svg?longCache=true&style=flat)](https://codecov.io/gh/MarcYin/SIAC) +[![Coverage Status](https://coveralls.io/repos/github/MarcYin/SIAC/badge.svg?branch=master)](https://coveralls.io/github/MarcYin/SIAC?branch=master) +[![Lisence](https://img.shields.io/pypi/l/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) + +This atmospheric correction method uses MODIS MCD43 BRDF product to get a coarse resolution simulation of earth surface. A model based on MODIS PSF is built to deal with the scale differences between MODIS and Sentinel 2 / Landsat 8. We uses the ECMWF CAMS prediction as a prior for the atmospheric states, coupling with 6S model to solve for the atmospheric parameters. We do not have topography correction and homogeneouse surface is used without considering the BRDF effects. + + +## Data needed: +* MCD43 : 16 days before and 16 days after the Sentinel 2 / Landsat 8 sensing date +* ECMWF CAMS Near Real Time prediction: a time step of 3 hours with the start time of 00:00:00 over the date, and data from 01/04/2015 are mirrored in UCL server at: http://www2.geog.ucl.ac.uk/~ucfafyi/cams/ +* Global DEM: Global DEM VRT file built from ASTGTM2 DEM, and most of the DEM over land are mirrored in UCL server at: http://www2.geog.ucl.ac.uk/~ucfafyi/eles/ +* Emulators: emulators for atmospheric path reflectance, total transmitance and single scattering Albedo, and the emulators for Sentinel 2, Landsat 8 and MODIS trained with 6S.V2 can be found at: http://www2.geog.ucl.ac.uk/~ucfafyi/emus/ + +## Installation: + +Download this repositories either by `git clone git@ github.com:MarcYin/SIAC.git` or download the zip file and unzip it. In the main directory of it excute `pip install .` , or `pip install SIAC` or `conda install -c f0xy siac` + +To save your time for installing GDAL and lightgbm, please use: + +```bash +conda install -c conda-forge gdal=2.2.4 +conda install -c conda-forge lightgbm +``` +
+The typical usage for **Sentinel 2**: + +```python +from SIAC import SIAC_S2 +SIAC_S2('/directory/where/you/store/S2/data/') # AWS or Senitinel offical package +``` +
+and **Landsat 8**: + +```python +from SIAC import SIAC_L8 +SIAC_L8('/directory/where/you/store/L8/data/') +``` +
+An example of correction for Landsat 5 for a more detailed demostration of the usage is shown [here](https://github.com/MarcYin/Global-analysis-ready-dataset) + +## Examples and Map: + +A [page](http://www2.geog.ucl.ac.uk/~ucfafyi/Atmo_Cor/index.html) shows some correction samples. + +A [map](http://www2.geog.ucl.ac.uk/~ucfafyi/map) for comparison between TOA and BOA. + +### LICENSE +GNU GENERAL PUBLIC LICENSE V3 From d3fd39d581a5e7c2166d5ce0685f95124b741f09 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Fri, 21 Sep 2018 19:00:25 +0100 Subject: [PATCH 038/103] add s2 test --- .gitignore | 1 + .travis.yml | 2 +- SIAC/tests/md5Checksum | 87 ++++++++++++++++++++++++++++++++++++++++ SIAC/tests/s2_flists.txt | 87 ++++++++++++++++++++++++++++++++++++++++ SIAC/tests/test_s2.py | 53 ++++++++++++++++++++++++ 5 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 SIAC/tests/md5Checksum create mode 100644 SIAC/tests/s2_flists.txt create mode 100644 SIAC/tests/test_s2.py diff --git a/.gitignore b/.gitignore index f70cbd0..1752046 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ *.hdf *.pkl !*/sen2cloud_detector.pkl +SIAC/tests/S2?_MSIL1C*.SAFE diff --git a/.travis.yml b/.travis.yml index 55699e9..52eeca7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ install: - conda config --set always_yes yes --set changeps1 no - conda config --add channels conda-forge - conda update -q conda - - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six lightgbm requests scikit-learn scikit-image gdal coveralls pytest-cov coverage=4.5.1 + - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six lightgbm requests scikit-learn scikit-image gdal coveralls pytest-cov coverage=4.5.1 h5py - source activate test-environment - conda info -a - conda list diff --git a/SIAC/tests/md5Checksum b/SIAC/tests/md5Checksum new file mode 100644 index 0000000..f26fff8 --- /dev/null +++ b/SIAC/tests/md5Checksum @@ -0,0 +1,87 @@ +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/MTD_TL.xml b629e33a460fd7d3877ff4f0e2e25e11 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/FORMAT_CORRECTNESS.xml 908efb8f4a4bf880c58b03b5434be94a +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/GENERAL_QUALITY.xml bb94215ad443a5b039f43134f7f20853 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/GEOMETRIC_QUALITY.xml f5689a6d2b33689fa1cb22c67f5232a7 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_CLOUDS_B00.gml 01334764d0fdd1dfcf9fd038f42d9878 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B01.gml 173711059dacc1613871be1b5ee50276 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B02.gml 66587815378fbfd0287f77aaec6c5e31 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B03.gml 6ce8412ad5c67bf855a5625b20b579a1 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B04.gml a407c9eccb4c596055c3d4992f821e94 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B05.gml b9665a92a46ea69e5cc64037aea0f0f9 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B06.gml 29c50809584b40d3a74be3bcac451d1a +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B07.gml a5593eaad8b0fa5357914d83c3b703a9 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B08.gml cba00daf654b8f743e05aba072f50799 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B09.gml 317cab0e885f4bbf2f497acbb6e64f02 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B10.gml c183d2fdb7a65e841912ca1d2e6d5581 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B11.gml 302cc6be50e3e16a6e3e0a778c1e97fb +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B12.gml c12d984a40242127490db55e3f02e731 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B8A.gml 5a8e29c7a666ca57b52fbf1edb154de8 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B01.gml 1df22d33efdc639f4a493b2e0e1b8d64 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B02.gml 1b15fbcfa8c2ad7b855752d5a7f6f59d +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B03.gml 6c0931611afa46d1bd5eb8e55845068a +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B04.gml b65b88e0b1e4ec91aa94b64c28b06f9a +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B05.gml bcd19b7fbbca63b9d76fb60e576c31ef +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B06.gml 8156a3a5ad3578a69669540de47a7cf1 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B07.gml 5e72c28f16fdc394ab9e3ccaf40f8b8f +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B08.gml e9ca087423faf7adace20d323f6cdf02 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B09.gml 12fe3be8331a3b1f43ff849c9b973b71 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B10.gml f500b1ba7e35bf0ddcf0c2ed2be3918a +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B11.gml 6e4938079885fec36ea49efa3727bb1c +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B12.gml 4a95fd680ebede38a37208dc7b54d06a +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B8A.gml 2331b2c18fad9ed04977976b58b4996a +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B01.gml 05cf377cf71a7313f7143e7c330d7403 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B02.gml 7fac91d6ec58de8e110b400f56f1e83f +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B03.gml 2a0a0cb056b3cae52dddaa18f89e7e48 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B04.gml af08f217bdf40ec5adab53fef04f0294 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B05.gml 27300154df0a196b02c21bccc54811e7 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B06.gml f05a0ea25550910a12dd2e4c87049ea0 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B07.gml ca344098b40167876cbb1eaae7af30e9 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B08.gml 23a834f36a2e35f1bb83b63f332a3212 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B09.gml 2a5c46cef481b397c6e23128ff7c69e8 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B10.gml 968464cfc8e5fe300b6e4691642d70ee +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B11.gml 70bdfee801685ffca1ff0500394d7262 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B12.gml e8d1efb5e96977ceb084c62a3c003338 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B8A.gml 2e4b24e43bc1895f23d4cd9c9d7c07fa +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B01.gml 19169f7c688bb0d9aaec829fd754ad97 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B02.gml 7ca31b958d0806bd1c711862a7f665be +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B03.gml 54df9169dd0b31cceb8f1b845596e1e0 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B04.gml de8f2666f2c328ee6ddba69764f88e39 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B05.gml be7c99202b7b5adefc9cdf81a1b02974 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B06.gml dbff626b6d531853b2a90207c3ed35bc +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B07.gml 27916122b5d54448e7ec3877c7930044 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B08.gml 6ba14c9236a8932a91a9ffb4a922dd9d +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B09.gml 230dac1bd0a705aaf1dfdb1dec1479ef +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B10.gml ffc7ba99d9cbfa2ebec8b3e1b9409ae8 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B11.gml eae1d306393194c5426fc0ac545a468c +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B12.gml db661c056b612ad1da9d8e1f7ef2bd75 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B8A.gml 10bfbf0129a2cafd543ca061936cc626 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B01.gml 9bd6cb96b3af8df23f7c8e6396175eb0 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B02.gml 2c8c24fa3750604202d10c409fa3d063 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B03.gml f6491e0c42a54eae0be23952e20309f8 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B04.gml b07c72b2b978436d589a0f1ad4d3f5f9 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B05.gml 9479787bd41a81f058dbc9cad84990b3 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B06.gml b08bad73a9b3f2883527807014f39341 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B07.gml fee117eac6d968c38b51017747ecac34 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B08.gml 86a2a510a972e804a8c6a0d36aed33cb +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B09.gml 82efbcba0f7a16f72d2e9dc154c68463 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B10.gml 38e194f2eb014f9256f248c56dadd144 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B11.gml 8c8ce44e1ca6964c24afbe3f7160161b +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B12.gml 4ab64a1f3440b4f706e3ea7567f14224 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B8A.gml 3a65a8c938b7fd741f42a20bf9f70b36 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/SENSOR_QUALITY.xml c4ec8f99f320500da59bfbeaa1015454 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/T31UCU_20171123T111349_PVI.jp2 610acb5d35f9adefcc37185d9a102fb2 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/AUX_DATA/AUX_ECMWFT 3bf4e5f5dfbf5bdf54f8a62d2fba6e50 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B01.jp2 f3272a3e952764c047128c35db24cc27 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B02.jp2 2f692a0105077bd8266c8c27a95dc239 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B03.jp2 e8d3d4e2d6c68b76fde511cdff459586 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B04.jp2 90509b0ac59ae4aacd5cbabd8f4fa7b8 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B05.jp2 91f90b797fe232add8c2cd8aaf3249f7 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B06.jp2 1ee4b649c9f8567d93d3bb293a6ec39d +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B07.jp2 e138256cea0daeef6160a8148fec0cce +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B08.jp2 1b88f6887c58684de73b4a612c127493 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B09.jp2 6ec09adae58369b1fda8caad5e6f321c +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B10.jp2 366b1dcae468e41b9cc342b66cb3bf2f +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B11.jp2 33cdfc4c645d5f281cacfdd9f5fbc8b7 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B12.jp2 106acf0603867c3ec74af9f2b2b71020 +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B8A.jp2 d0a9537e01530bbf372338cf3585171a +S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_TCI.jp2 8d2d39600d1b3d6ba2e3978e7f52fc68 diff --git a/SIAC/tests/s2_flists.txt b/SIAC/tests/s2_flists.txt new file mode 100644 index 0000000..3e937dc --- /dev/null +++ b/SIAC/tests/s2_flists.txt @@ -0,0 +1,87 @@ +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/MTD_TL.xml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/FORMAT_CORRECTNESS.xml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/GENERAL_QUALITY.xml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/GEOMETRIC_QUALITY.xml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_CLOUDS_B00.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B01.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B02.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B03.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B04.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B05.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B06.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B07.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B08.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B09.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B10.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B11.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B12.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B8A.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B01.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B02.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B03.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B04.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B05.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B06.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B07.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B08.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B09.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B10.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B11.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B12.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B8A.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B01.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B02.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B03.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B04.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B05.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B06.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B07.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B08.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B09.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B10.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B11.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B12.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B8A.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B01.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B02.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B03.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B04.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B05.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B06.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B07.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B08.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B09.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B10.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B11.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B12.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B8A.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B01.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B02.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B03.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B04.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B05.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B06.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B07.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B08.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B09.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B10.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B11.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B12.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B8A.gml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/SENSOR_QUALITY.xml +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/T31UCU_20171123T111349_PVI.jp2 +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/AUX_DATA/AUX_ECMWFT +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B01.jp2 +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B02.jp2 +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B03.jp2 +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B04.jp2 +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B05.jp2 +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B06.jp2 +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B07.jp2 +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B08.jp2 +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B09.jp2 +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B10.jp2 +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B11.jp2 +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B12.jp2 +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B8A.jp2 +https://storage.googleapis.com/gcp-public-data-sentinel-2/tiles/31/U/CU/S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_TCI.jp2 diff --git a/SIAC/tests/test_s2.py b/SIAC/tests/test_s2.py new file mode 100644 index 0000000..27980fe --- /dev/null +++ b/SIAC/tests/test_s2.py @@ -0,0 +1,53 @@ +import os, sys +import requests +import numpy as np + +myPath = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, myPath + '/../../') +user_pass = os.environ['Earthdata_user'], os.environ['Earthdata_pass'] + +with open(myPath.replace('tests', '') + 'data/.earthdata_auth', 'wb') as f: + for i in user_pass: + f.write((i+'\n').encode()) + +from SIAC import SIAC_S2 + +with open('s2_flists.txt', 'rb') as f: + urls = [i.split('\n')[0].encode() for i in f.readlines()] + +import hashlib +def file_as_bytes(file): + with file: + return file.read() + +for url in urls: + filename = '/'.join(url.split('/')[8:]) + if not os.path.exists(filename): + req = requests.get(url) + if req.ok: + print('downloading %s' % filename) + if not os.path.exists(os.path.dirname(filename)): + try: + os.makedirs(os.path.dirname(filename)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + with open(filename, "w") as f: + f.write(req.content) + else: + pass + +flists_md5 = np.loadtxt('md5Checksum', dtype='str') + +for f_md5 in flists_md5: + filename, md5 = f_md5 + if os.path.exists(filename): + assert (hashlib.md5(file_as_bytes(open(filename, 'rb'))).hexdigest() == md5, 'file size is not right') + else: + raise IOError +s2_file_dir = filename.split('/')[0] +SIAC_S2(s2_file_dir) + + +with open(myPath.replace('tests', '') + 'data/.earthdata_auth', 'wb') as f: + f.write(('').encode()) From 56aab4f2ffd59518463565f9236302ecd803ec6a Mon Sep 17 00:00:00 2001 From: MarcYin Date: Sat, 22 Sep 2018 00:28:46 +0100 Subject: [PATCH 039/103] bug --- SIAC/tests/test_s2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SIAC/tests/test_s2.py b/SIAC/tests/test_s2.py index 27980fe..21b7470 100644 --- a/SIAC/tests/test_s2.py +++ b/SIAC/tests/test_s2.py @@ -12,7 +12,7 @@ from SIAC import SIAC_S2 -with open('s2_flists.txt', 'rb') as f: +with open(myPath + '/s2_flists.txt', 'rb') as f: urls = [i.split('\n')[0].encode() for i in f.readlines()] import hashlib From 3f6867f099ccb95f46097b287fbe001f15bfc480 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Sat, 22 Sep 2018 13:33:13 +0100 Subject: [PATCH 040/103] debug --- SIAC/tests/test_s2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SIAC/tests/test_s2.py b/SIAC/tests/test_s2.py index 21b7470..0fa8472 100644 --- a/SIAC/tests/test_s2.py +++ b/SIAC/tests/test_s2.py @@ -13,7 +13,7 @@ from SIAC import SIAC_S2 with open(myPath + '/s2_flists.txt', 'rb') as f: - urls = [i.split('\n')[0].encode() for i in f.readlines()] + urls = [str(i.split('\n')[0]) for i in f.readlines()] import hashlib def file_as_bytes(file): @@ -37,7 +37,7 @@ def file_as_bytes(file): else: pass -flists_md5 = np.loadtxt('md5Checksum', dtype='str') +flists_md5 = np.loadtxt(myPath + '/md5Checksum', dtype='str') for f_md5 in flists_md5: filename, md5 = f_md5 From d7869451b24db02240c463128e12385d72f7da42 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Sat, 22 Sep 2018 15:02:22 +0100 Subject: [PATCH 041/103] debug... --- .travis.yml | 2 +- SIAC/tests/test_s2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 52eeca7..d73679a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ install: - conda config --set always_yes yes --set changeps1 no - conda config --add channels conda-forge - conda update -q conda - - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six lightgbm requests scikit-learn scikit-image gdal coveralls pytest-cov coverage=4.5.1 h5py + - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six lightgbm requests scikit-learn scikit-image gdal=2.2.4 coveralls pytest-cov coverage=4.5.1 h5py - source activate test-environment - conda info -a - conda list diff --git a/SIAC/tests/test_s2.py b/SIAC/tests/test_s2.py index 0fa8472..f2b3dc5 100644 --- a/SIAC/tests/test_s2.py +++ b/SIAC/tests/test_s2.py @@ -13,7 +13,7 @@ from SIAC import SIAC_S2 with open(myPath + '/s2_flists.txt', 'rb') as f: - urls = [str(i.split('\n')[0]) for i in f.readlines()] + urls = [(i.split('\n')[0]).decode() for i in f.readlines()] import hashlib def file_as_bytes(file): From 662432a8ac41182032e4d191e7800e82e9aa23d9 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Sat, 22 Sep 2018 18:27:37 +0100 Subject: [PATCH 042/103] debug... --- SIAC/tests/test_s2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SIAC/tests/test_s2.py b/SIAC/tests/test_s2.py index f2b3dc5..eed672d 100644 --- a/SIAC/tests/test_s2.py +++ b/SIAC/tests/test_s2.py @@ -13,7 +13,7 @@ from SIAC import SIAC_S2 with open(myPath + '/s2_flists.txt', 'rb') as f: - urls = [(i.split('\n')[0]).decode() for i in f.readlines()] + urls = [i.split('\n')[0] for i in f.readlines()] import hashlib def file_as_bytes(file): From a6dc0ea2a17026643186373946682133a7a0b642 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Mon, 24 Sep 2018 10:33:08 +0100 Subject: [PATCH 043/103] debug.. --- SIAC/tests/test_s2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SIAC/tests/test_s2.py b/SIAC/tests/test_s2.py index eed672d..c725ebb 100644 --- a/SIAC/tests/test_s2.py +++ b/SIAC/tests/test_s2.py @@ -13,7 +13,7 @@ from SIAC import SIAC_S2 with open(myPath + '/s2_flists.txt', 'rb') as f: - urls = [i.split('\n')[0] for i in f.readlines()] + urls = [i.split(b'\n')[0] for i in f.readlines()] import hashlib def file_as_bytes(file): From d8ff53c15ab4f2835f6277e5c0099fcaab288f1b Mon Sep 17 00:00:00 2001 From: MarcYin Date: Mon, 24 Sep 2018 12:21:25 +0100 Subject: [PATCH 044/103] test stdout --- .gitignore | 2 +- .travis.yml | 2 +- SIAC/tests/test_s2.py | 84 ++++++++++++++++++++++--------------------- 3 files changed, 45 insertions(+), 43 deletions(-) diff --git a/.gitignore b/.gitignore index 1752046..e6537f0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ *.hdf *.pkl !*/sen2cloud_detector.pkl -SIAC/tests/S2?_MSIL1C*.SAFE +*S2?_MSIL1C*.SAFE diff --git a/.travis.yml b/.travis.yml index d73679a..ffd79b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ install: - pip install coveralls - pip install codecov script: - - py.test --cov + - py.test -s --cov - coverage xml after_success: - codecov diff --git a/SIAC/tests/test_s2.py b/SIAC/tests/test_s2.py index c725ebb..8c500f6 100644 --- a/SIAC/tests/test_s2.py +++ b/SIAC/tests/test_s2.py @@ -10,44 +10,46 @@ for i in user_pass: f.write((i+'\n').encode()) -from SIAC import SIAC_S2 - -with open(myPath + '/s2_flists.txt', 'rb') as f: - urls = [i.split(b'\n')[0] for i in f.readlines()] - -import hashlib -def file_as_bytes(file): - with file: - return file.read() - -for url in urls: - filename = '/'.join(url.split('/')[8:]) - if not os.path.exists(filename): - req = requests.get(url) - if req.ok: - print('downloading %s' % filename) - if not os.path.exists(os.path.dirname(filename)): - try: - os.makedirs(os.path.dirname(filename)) - except OSError as exc: # Guard against race condition - if exc.errno != errno.EEXIST: - raise - with open(filename, "w") as f: - f.write(req.content) - else: - pass - -flists_md5 = np.loadtxt(myPath + '/md5Checksum', dtype='str') - -for f_md5 in flists_md5: - filename, md5 = f_md5 - if os.path.exists(filename): - assert (hashlib.md5(file_as_bytes(open(filename, 'rb'))).hexdigest() == md5, 'file size is not right') - else: - raise IOError -s2_file_dir = filename.split('/')[0] -SIAC_S2(s2_file_dir) - - -with open(myPath.replace('tests', '') + 'data/.earthdata_auth', 'wb') as f: - f.write(('').encode()) +def test_s2(): + from SIAC import SIAC_S2 + with open(myPath + '/s2_flists.txt', 'rb') as f: + urls = [i.split(b'\n')[0] for i in f.readlines()] + + import hashlib + def file_as_bytes(file): + with file: + return file.read() + + for url in urls: + url = str(url) + filename = '/'.join(url.split('/')[8:]) + if not os.path.exists(filename): + req = requests.get(url) + if req.ok: + print('downloading %s' % filename) + if not os.path.exists(os.path.dirname(filename)): + try: + os.makedirs(os.path.dirname(filename)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + with open(filename, "w") as f: + f.write(req.content) + else: + pass + + flists_md5 = np.loadtxt(myPath + '/md5Checksum', dtype='str') + + for f_md5 in flists_md5: + filename, md5 = f_md5 + if os.path.exists(filename): + assert (hashlib.md5(file_as_bytes(open(filename, 'rb'))).hexdigest() == md5, 'file size is not right') + else: + raise IOError + + s2_file_dir = filename.split('/')[0] + SIAC_S2(s2_file_dir) + + with open(myPath.replace('tests', '') + 'data/.earthdata_auth', 'wb') as f: + f.write(('').encode()) + assert True From 6f779a53e5fb647cd06c62f4e1f6ab3cb8b448fa Mon Sep 17 00:00:00 2001 From: MarcYin Date: Mon, 24 Sep 2018 12:38:57 +0100 Subject: [PATCH 045/103] string again! --- SIAC/tests/test_s2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SIAC/tests/test_s2.py b/SIAC/tests/test_s2.py index 8c500f6..f5b27cd 100644 --- a/SIAC/tests/test_s2.py +++ b/SIAC/tests/test_s2.py @@ -13,7 +13,7 @@ def test_s2(): from SIAC import SIAC_S2 with open(myPath + '/s2_flists.txt', 'rb') as f: - urls = [i.split(b'\n')[0] for i in f.readlines()] + urls = [i.encode().split('\n')[0] for i in f.readlines()] import hashlib def file_as_bytes(file): @@ -21,7 +21,7 @@ def file_as_bytes(file): return file.read() for url in urls: - url = str(url) + url = str(url.encode()) filename = '/'.join(url.split('/')[8:]) if not os.path.exists(filename): req = requests.get(url) From 946760a0870a32d565473729ff67655eb7bcea68 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Mon, 24 Sep 2018 12:55:12 +0100 Subject: [PATCH 046/103] bytes unicode or something else.... --- SIAC/tests/test_s2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SIAC/tests/test_s2.py b/SIAC/tests/test_s2.py index f5b27cd..7832f1f 100644 --- a/SIAC/tests/test_s2.py +++ b/SIAC/tests/test_s2.py @@ -13,7 +13,7 @@ def test_s2(): from SIAC import SIAC_S2 with open(myPath + '/s2_flists.txt', 'rb') as f: - urls = [i.encode().split('\n')[0] for i in f.readlines()] + urls = [i.decode().split('\n')[0] for i in f.readlines()] import hashlib def file_as_bytes(file): @@ -21,7 +21,7 @@ def file_as_bytes(file): return file.read() for url in urls: - url = str(url.encode()) + url = str(url.decode()) filename = '/'.join(url.split('/')[8:]) if not os.path.exists(filename): req = requests.get(url) From fcfc7e2eefe3779912e0b7b200045a83b83a4a79 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Mon, 24 Sep 2018 13:19:14 +0100 Subject: [PATCH 047/103] debug... --- SIAC/get_MCD43.py | 5 ++++- SIAC/tests/test_import.py | 15 --------------- SIAC/tests/test_s2.py | 2 +- 3 files changed, 5 insertions(+), 17 deletions(-) delete mode 100644 SIAC/tests/test_import.py diff --git a/SIAC/get_MCD43.py b/SIAC/get_MCD43.py index 6639212..e1fb0ca 100644 --- a/SIAC/get_MCD43.py +++ b/SIAC/get_MCD43.py @@ -41,7 +41,10 @@ for i in up: f.write((i + '\n').encode()) auth = tuple([username, password]) - + +#print(file_path + '/data/.earthdata_auth') +#print(auth) + def find_files(aoi, obs_time, temporal_window = 16): days = [(obs_time - timedelta(days = int(i))).strftime('%Y.%m.%d') for i in np.arange(temporal_window, 0, -1)] + \ [(obs_time + timedelta(days = int(i))).strftime('%Y.%m.%d') for i in np.arange(0, temporal_window+1, 1)] diff --git a/SIAC/tests/test_import.py b/SIAC/tests/test_import.py deleted file mode 100644 index afa8ade..0000000 --- a/SIAC/tests/test_import.py +++ /dev/null @@ -1,15 +0,0 @@ -import sys, os -myPath = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, myPath + '/../../') -user_pass = 'user', 'pass' - -with open(myPath.replace('tests', '') + 'data/.earthdata_auth', 'wb') as f: - for i in user_pass: - f.write((i+'\n').encode()) - -from SIAC import SIAC_S2 -from SIAC import SIAC_L8 - - -with open(myPath.replace('tests', '') + 'data/.earthdata_auth', 'wb') as f: - f.write(('').encode()) diff --git a/SIAC/tests/test_s2.py b/SIAC/tests/test_s2.py index 7832f1f..0d6c8c9 100644 --- a/SIAC/tests/test_s2.py +++ b/SIAC/tests/test_s2.py @@ -21,7 +21,7 @@ def file_as_bytes(file): return file.read() for url in urls: - url = str(url.decode()) + url = url filename = '/'.join(url.split('/')[8:]) if not os.path.exists(filename): req = requests.get(url) From c489cdb915873f85e423d307c3a96275e20dd300 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Mon, 24 Sep 2018 13:29:14 +0100 Subject: [PATCH 048/103] string still... --- SIAC/tests/test_s2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SIAC/tests/test_s2.py b/SIAC/tests/test_s2.py index 0d6c8c9..8890730 100644 --- a/SIAC/tests/test_s2.py +++ b/SIAC/tests/test_s2.py @@ -33,7 +33,7 @@ def file_as_bytes(file): except OSError as exc: # Guard against race condition if exc.errno != errno.EEXIST: raise - with open(filename, "w") as f: + with open(filename, "wb") as f: f.write(req.content) else: pass From 1b54fabb44c6712645fe290fb0c4f45271637b7e Mon Sep 17 00:00:00 2001 From: MarcYin Date: Mon, 24 Sep 2018 15:25:27 +0100 Subject: [PATCH 049/103] try to debug requests --- .gitignore | 1 + SIAC/get_MCD43.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e6537f0..c4b7175 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ *.pkl !*/sen2cloud_detector.pkl *S2?_MSIL1C*.SAFE +.coverage* diff --git a/SIAC/get_MCD43.py b/SIAC/get_MCD43.py index e1fb0ca..0ff0bec 100644 --- a/SIAC/get_MCD43.py +++ b/SIAC/get_MCD43.py @@ -80,7 +80,7 @@ def downloader(url_fname): with requests.Session() as s: s.auth = auth r1 = s.get(url) - r = s.get(r1.url, stream=True) + r = s.get(r1.url, stream=True, headers={'user-agent': 'My app'}) if r.ok: remote_size = int(r.headers['Content-Length']) if os.path.exists(fname): From 4263a7e365882198ac763b5797a33d400cc01d3b Mon Sep 17 00:00:00 2001 From: MarcYin Date: Mon, 24 Sep 2018 16:09:05 +0100 Subject: [PATCH 050/103] some debug --- SIAC/get_MCD43.py | 1 + 1 file changed, 1 insertion(+) diff --git a/SIAC/get_MCD43.py b/SIAC/get_MCD43.py index 0ff0bec..6828c95 100644 --- a/SIAC/get_MCD43.py +++ b/SIAC/get_MCD43.py @@ -78,6 +78,7 @@ def get_one_tile(tile_date): def downloader(url_fname): url, fname = url_fname with requests.Session() as s: + s.max_redirects = 100 s.auth = auth r1 = s.get(url) r = s.get(r1.url, stream=True, headers={'user-agent': 'My app'}) From 0b4275ceacd13beba06fe618d3cd50bf7b4579e3 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Mon, 24 Sep 2018 16:42:45 +0100 Subject: [PATCH 051/103] debug --- SIAC/SIAC_L8.py | 2 +- SIAC/SIAC_S2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SIAC/SIAC_L8.py b/SIAC/SIAC_L8.py index 16970f6..8197100 100644 --- a/SIAC/SIAC_L8.py +++ b/SIAC/SIAC_L8.py @@ -80,7 +80,7 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, qa_name, cloud_mask, \ off = off / np.cos(np.deg2rad(sza)) aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,sun_angles,\ obs_time,cloud_mask, gamma=10., ref_scale = scale, ref_off = off, \ - spec_m_dir=file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/') + spec_m_dir=file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', mcd43_dir=vrt_dir) aero._solving() toa_bands = toa_refs view_angles = view_ang_names diff --git a/SIAC/SIAC_S2.py b/SIAC/SIAC_S2.py index 06764a3..00b9ba3 100644 --- a/SIAC/SIAC_S2.py +++ b/SIAC/SIAC_S2.py @@ -70,7 +70,7 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, \ sun_angles = sun_ang_name aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,\ sun_angles,obs_time,cloud_mask, gamma=10., spec_m_dir= \ - file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/') + file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', mcd43_dir=vrt_dir) aero._solving() toa_bands = toa_refs view_angles = view_ang_names From 31acefb8f9d3c1bd083776e8f3244fe488263177 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Mon, 24 Sep 2018 17:15:45 +0100 Subject: [PATCH 052/103] should work now.... --- SIAC/SIAC_L8.py | 16 ++++++++-------- SIAC/SIAC_S2.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/SIAC/SIAC_L8.py b/SIAC/SIAC_L8.py index 8197100..b70aced 100644 --- a/SIAC/SIAC_L8.py +++ b/SIAC/SIAC_L8.py @@ -15,7 +15,7 @@ home = expanduser("~") file_path = os.path.dirname(os.path.realpath(__file__)) -def SIAC_L8(l8_dir, send_back = False, mcd43 = home + '/MCD43', vrt_dir = home + '/MCD43_VRT'): +def SIAC_L8(l8_dir, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/'): file_path = os.path.dirname(os.path.realpath(__file__)) if not os.path.exists(file_path + '/emus/'): os.mkdir(file_path + '/emus/') @@ -44,15 +44,15 @@ def SIAC_L8(l8_dir, send_back = False, mcd43 = home + '/MCD43', vrt_dir = home + return aero_atmos def do_correction(sun_ang_name, view_ang_names, toa_refs, qa_name, cloud_mask, \ - metafile, mcd43 = home + '/MCD43', vrt_dir = home + '/MCD43_VRT'): + metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/'): - if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43'): - if not os.path.exists(home + '/MCD43'): - os.mkdir(home + '/MCD43') + if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'): + if not os.path.exists(home + '/MCD43/'): + os.mkdir(home + '/MCD43/') - if os.path.realpath(vrt_dir) in os.path.realpath(home + '/MCD43_VRT'): - if not os.path.exists(home + '/MCD43_VRT'): - os.mkdir(home + '/MCD43_VRT') + if os.path.realpath(vrt_dir) in os.path.realpath(home + '/MCD43_VRT/'): + if not os.path.exists(home + '/MCD43_VRT/'): + os.mkdir(home + '/MCD43_VRT/') base = os.path.dirname(toa_refs[0]) with open(metafile) as f: diff --git a/SIAC/SIAC_S2.py b/SIAC/SIAC_S2.py index 00b9ba3..3e42254 100644 --- a/SIAC/SIAC_S2.py +++ b/SIAC/SIAC_S2.py @@ -14,7 +14,7 @@ home = expanduser("~") file_path = os.path.dirname(os.path.realpath(__file__)) -def SIAC_S2(s2_t, send_back = False, mcd43 = home + '/MCD43', vrt_dir = home + '/MCD43_VRT'): +def SIAC_S2(s2_t, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/'): if not os.path.exists(file_path + '/emus/'): os.mkdir(file_path + '/emus/') if len(glob(file_path + '/emus/' + 'isotropic_MSI_emulators_*_x?p_S2?.pkl')) < 12: @@ -40,15 +40,15 @@ def SIAC_S2(s2_t, send_back = False, mcd43 = home + '/MCD43', vrt_dir = home + ' return aero_atmos def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, \ - cloud_mask, metafile, mcd43 = home + '/MCD43', vrt_dir = home + '/MCD43_VRT'): + cloud_mask, metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/'): - if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43'): - if not os.path.exists(home + '/MCD43'): - os.mkdir(home + '/MCD43') + if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'): + if not os.path.exists(home + '/MCD43/'): + os.mkdir(home + '/MCD43/') - if os.path.realpath(vrt_dir) in os.path.realpath(home + '/MCD43_VRT'): - if not os.path.exists(home + '/MCD43_VRT'): - os.mkdir(home + '/MCD43_VRT') + if os.path.realpath(vrt_dir) in os.path.realpath(home + '/MCD43_VRT/'): + if not os.path.exists(home + '/MCD43_VRT/'): + os.mkdir(home + '/MCD43_VRT/') base = os.path.dirname(toa_refs[0]) base = toa_refs[0].replace('B01.jp2', '') From 24268bcddc5d1efab833de89e6d1aedb6ce7dd43 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Mon, 24 Sep 2018 17:45:45 +0100 Subject: [PATCH 053/103] delete checksum --- SIAC/tests/md5Checksum | 87 ------------------------------------------ SIAC/tests/test_s2.py | 9 ----- 2 files changed, 96 deletions(-) delete mode 100644 SIAC/tests/md5Checksum diff --git a/SIAC/tests/md5Checksum b/SIAC/tests/md5Checksum deleted file mode 100644 index f26fff8..0000000 --- a/SIAC/tests/md5Checksum +++ /dev/null @@ -1,87 +0,0 @@ -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/MTD_TL.xml b629e33a460fd7d3877ff4f0e2e25e11 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/FORMAT_CORRECTNESS.xml 908efb8f4a4bf880c58b03b5434be94a -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/GENERAL_QUALITY.xml bb94215ad443a5b039f43134f7f20853 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/GEOMETRIC_QUALITY.xml f5689a6d2b33689fa1cb22c67f5232a7 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_CLOUDS_B00.gml 01334764d0fdd1dfcf9fd038f42d9878 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B01.gml 173711059dacc1613871be1b5ee50276 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B02.gml 66587815378fbfd0287f77aaec6c5e31 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B03.gml 6ce8412ad5c67bf855a5625b20b579a1 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B04.gml a407c9eccb4c596055c3d4992f821e94 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B05.gml b9665a92a46ea69e5cc64037aea0f0f9 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B06.gml 29c50809584b40d3a74be3bcac451d1a -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B07.gml a5593eaad8b0fa5357914d83c3b703a9 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B08.gml cba00daf654b8f743e05aba072f50799 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B09.gml 317cab0e885f4bbf2f497acbb6e64f02 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B10.gml c183d2fdb7a65e841912ca1d2e6d5581 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B11.gml 302cc6be50e3e16a6e3e0a778c1e97fb -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B12.gml c12d984a40242127490db55e3f02e731 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DEFECT_B8A.gml 5a8e29c7a666ca57b52fbf1edb154de8 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B01.gml 1df22d33efdc639f4a493b2e0e1b8d64 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B02.gml 1b15fbcfa8c2ad7b855752d5a7f6f59d -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B03.gml 6c0931611afa46d1bd5eb8e55845068a -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B04.gml b65b88e0b1e4ec91aa94b64c28b06f9a -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B05.gml bcd19b7fbbca63b9d76fb60e576c31ef -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B06.gml 8156a3a5ad3578a69669540de47a7cf1 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B07.gml 5e72c28f16fdc394ab9e3ccaf40f8b8f -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B08.gml e9ca087423faf7adace20d323f6cdf02 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B09.gml 12fe3be8331a3b1f43ff849c9b973b71 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B10.gml f500b1ba7e35bf0ddcf0c2ed2be3918a -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B11.gml 6e4938079885fec36ea49efa3727bb1c -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B12.gml 4a95fd680ebede38a37208dc7b54d06a -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_DETFOO_B8A.gml 2331b2c18fad9ed04977976b58b4996a -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B01.gml 05cf377cf71a7313f7143e7c330d7403 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B02.gml 7fac91d6ec58de8e110b400f56f1e83f -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B03.gml 2a0a0cb056b3cae52dddaa18f89e7e48 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B04.gml af08f217bdf40ec5adab53fef04f0294 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B05.gml 27300154df0a196b02c21bccc54811e7 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B06.gml f05a0ea25550910a12dd2e4c87049ea0 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B07.gml ca344098b40167876cbb1eaae7af30e9 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B08.gml 23a834f36a2e35f1bb83b63f332a3212 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B09.gml 2a5c46cef481b397c6e23128ff7c69e8 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B10.gml 968464cfc8e5fe300b6e4691642d70ee -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B11.gml 70bdfee801685ffca1ff0500394d7262 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B12.gml e8d1efb5e96977ceb084c62a3c003338 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_NODATA_B8A.gml 2e4b24e43bc1895f23d4cd9c9d7c07fa -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B01.gml 19169f7c688bb0d9aaec829fd754ad97 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B02.gml 7ca31b958d0806bd1c711862a7f665be -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B03.gml 54df9169dd0b31cceb8f1b845596e1e0 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B04.gml de8f2666f2c328ee6ddba69764f88e39 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B05.gml be7c99202b7b5adefc9cdf81a1b02974 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B06.gml dbff626b6d531853b2a90207c3ed35bc -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B07.gml 27916122b5d54448e7ec3877c7930044 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B08.gml 6ba14c9236a8932a91a9ffb4a922dd9d -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B09.gml 230dac1bd0a705aaf1dfdb1dec1479ef -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B10.gml ffc7ba99d9cbfa2ebec8b3e1b9409ae8 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B11.gml eae1d306393194c5426fc0ac545a468c -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B12.gml db661c056b612ad1da9d8e1f7ef2bd75 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_SATURA_B8A.gml 10bfbf0129a2cafd543ca061936cc626 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B01.gml 9bd6cb96b3af8df23f7c8e6396175eb0 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B02.gml 2c8c24fa3750604202d10c409fa3d063 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B03.gml f6491e0c42a54eae0be23952e20309f8 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B04.gml b07c72b2b978436d589a0f1ad4d3f5f9 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B05.gml 9479787bd41a81f058dbc9cad84990b3 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B06.gml b08bad73a9b3f2883527807014f39341 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B07.gml fee117eac6d968c38b51017747ecac34 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B08.gml 86a2a510a972e804a8c6a0d36aed33cb -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B09.gml 82efbcba0f7a16f72d2e9dc154c68463 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B10.gml 38e194f2eb014f9256f248c56dadd144 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B11.gml 8c8ce44e1ca6964c24afbe3f7160161b -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B12.gml 4ab64a1f3440b4f706e3ea7567f14224 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/MSK_TECQUA_B8A.gml 3a65a8c938b7fd741f42a20bf9f70b36 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/SENSOR_QUALITY.xml c4ec8f99f320500da59bfbeaa1015454 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/QI_DATA/T31UCU_20171123T111349_PVI.jp2 610acb5d35f9adefcc37185d9a102fb2 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/AUX_DATA/AUX_ECMWFT 3bf4e5f5dfbf5bdf54f8a62d2fba6e50 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B01.jp2 f3272a3e952764c047128c35db24cc27 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B02.jp2 2f692a0105077bd8266c8c27a95dc239 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B03.jp2 e8d3d4e2d6c68b76fde511cdff459586 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B04.jp2 90509b0ac59ae4aacd5cbabd8f4fa7b8 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B05.jp2 91f90b797fe232add8c2cd8aaf3249f7 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B06.jp2 1ee4b649c9f8567d93d3bb293a6ec39d -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B07.jp2 e138256cea0daeef6160a8148fec0cce -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B08.jp2 1b88f6887c58684de73b4a612c127493 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B09.jp2 6ec09adae58369b1fda8caad5e6f321c -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B10.jp2 366b1dcae468e41b9cc342b66cb3bf2f -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B11.jp2 33cdfc4c645d5f281cacfdd9f5fbc8b7 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B12.jp2 106acf0603867c3ec74af9f2b2b71020 -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_B8A.jp2 d0a9537e01530bbf372338cf3585171a -S2B_MSIL1C_20171123T111349_N0206_R137_T31UCU_20171123T130706.SAFE/GRANULE/L1C_T31UCU_A003738_20171123T111347/IMG_DATA/T31UCU_20171123T111349_TCI.jp2 8d2d39600d1b3d6ba2e3978e7f52fc68 diff --git a/SIAC/tests/test_s2.py b/SIAC/tests/test_s2.py index 8890730..c04370e 100644 --- a/SIAC/tests/test_s2.py +++ b/SIAC/tests/test_s2.py @@ -38,15 +38,6 @@ def file_as_bytes(file): else: pass - flists_md5 = np.loadtxt(myPath + '/md5Checksum', dtype='str') - - for f_md5 in flists_md5: - filename, md5 = f_md5 - if os.path.exists(filename): - assert (hashlib.md5(file_as_bytes(open(filename, 'rb'))).hexdigest() == md5, 'file size is not right') - else: - raise IOError - s2_file_dir = filename.split('/')[0] SIAC_S2(s2_file_dir) From 8c093ac2f631999ba52cc5d929504dc0cefa043f Mon Sep 17 00:00:00 2001 From: MarcYin Date: Mon, 24 Sep 2018 18:04:40 +0100 Subject: [PATCH 054/103] god help... --- SIAC/s2_preprocessing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SIAC/s2_preprocessing.py b/SIAC/s2_preprocessing.py index 0e02e8d..1e3f2bb 100644 --- a/SIAC/s2_preprocessing.py +++ b/SIAC/s2_preprocessing.py @@ -22,7 +22,7 @@ def do_cloud(cloud_bands, cloud_name = None): - toas = [reproject_data(band, cloud_bands[0], dstNodata=0, resample=gdal.GRIORA_Average).data for band in cloud_bands] + toas = [reproject_data(band, cloud_bands[0], dstNodata=0, resample=5).data for band in cloud_bands] toas = np.array(toas)/10000. mask = np.all(toas >= 0.0001, axis=0) cloud_proba = cl.predict_proba(toas[:, mask].T) From 12aed858e92b9b99cc339d4aaea60e9d5955823b Mon Sep 17 00:00:00 2001 From: MarcYin Date: Mon, 24 Sep 2018 18:24:27 +0100 Subject: [PATCH 055/103] add more time... --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ffd79b0..60965fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ install: - pip install coveralls - pip install codecov script: - - py.test -s --cov + - travis_wait 60 py.test -s --cov - coverage xml after_success: - codecov From 85691e5b6342202d56f88f49d8818182f09df7ca Mon Sep 17 00:00:00 2001 From: MarcYin Date: Mon, 24 Sep 2018 18:44:24 +0100 Subject: [PATCH 056/103] add better sleep --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 60965fc..ebb8601 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,12 @@ install: - pip install coveralls - pip install codecov script: - - travis_wait 60 py.test -s --cov + # Output something every 10 minutes or Travis kills the job + - while sleep 30m; do echo "=====[ $SECONDS seconds still running ]====="; done & + - py.test -s --cov + #Killing background sleep loop + - kill %1 + #- travis_wait 60 py.test -s --cov - coverage xml after_success: - codecov From c445265a2ce19624412e61c0ddd7fcabd8258eaa Mon Sep 17 00:00:00 2001 From: MarcYin Date: Mon, 24 Sep 2018 19:05:33 +0100 Subject: [PATCH 057/103] ... --- SIAC/get_MCD43.py | 2 +- SIAC/s2_preprocessing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SIAC/get_MCD43.py b/SIAC/get_MCD43.py index 6828c95..d2f4328 100644 --- a/SIAC/get_MCD43.py +++ b/SIAC/get_MCD43.py @@ -78,7 +78,7 @@ def get_one_tile(tile_date): def downloader(url_fname): url, fname = url_fname with requests.Session() as s: - s.max_redirects = 100 + s.max_redirects = 100000 s.auth = auth r1 = s.get(url) r = s.get(r1.url, stream=True, headers={'user-agent': 'My app'}) diff --git a/SIAC/s2_preprocessing.py b/SIAC/s2_preprocessing.py index 1e3f2bb..ff354b1 100644 --- a/SIAC/s2_preprocessing.py +++ b/SIAC/s2_preprocessing.py @@ -22,7 +22,7 @@ def do_cloud(cloud_bands, cloud_name = None): - toas = [reproject_data(band, cloud_bands[0], dstNodata=0, resample=5).data for band in cloud_bands] + toas = [reproject_data(str(band), cloud_bands[0], dstNodata=0, resample=5).data for band in cloud_bands] toas = np.array(toas)/10000. mask = np.all(toas >= 0.0001, axis=0) cloud_proba = cl.predict_proba(toas[:, mask].T) From f3597b7fef3e1ef5a828e9bc3f0aa5ee330ecc5a Mon Sep 17 00:00:00 2001 From: MarcYin Date: Tue, 25 Sep 2018 11:32:09 +0100 Subject: [PATCH 058/103] travis_wait --- .travis.yml | 2 +- SIAC/the_aerosol.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ebb8601..bf4df5c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ install: - pip install codecov script: # Output something every 10 minutes or Travis kills the job - - while sleep 30m; do echo "=====[ $SECONDS seconds still running ]====="; done & + - while sleep 9m; do echo "=====[ $SECONDS seconds still running ]====="; done & - py.test -s --cov #Killing background sleep loop - kill %1 diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index c5894f3..968f4a2 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -261,7 +261,7 @@ def _create_band_gs(self,): ''' self._toa_bands = [] for band in self.toa_bands: - g = gdal.Warp('', band, format = 'MEM', xRes = self.pixel_res, yRes = self.pixel_res, warpOptions = ['NUM_THREADS=ALL_CPUS'],\ + g = gdal.Warp('', str(band), format = 'MEM', xRes = self.pixel_res, yRes = self.pixel_res, warpOptions = ['NUM_THREADS=ALL_CPUS'],\ srcNodata = 0, dstNodata=0, cutlineDSName= self.aoi, cropToCutline=True, resampleAlg = 0) self._toa_bands.append(g) self.example_file = self._toa_bands[0] From 321e76cdcf757f9a2acf88fba8a829bf871bbcab Mon Sep 17 00:00:00 2001 From: MarcYin Date: Tue, 25 Sep 2018 12:39:53 +0100 Subject: [PATCH 059/103] add aoi --- SIAC/SIAC_L8.py | 8 ++++---- SIAC/SIAC_S2.py | 8 ++++---- SIAC/tests/aoi.geojson | 1 + SIAC/tests/test_s2.py | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) create mode 100644 SIAC/tests/aoi.geojson diff --git a/SIAC/SIAC_L8.py b/SIAC/SIAC_L8.py index b70aced..0a3c800 100644 --- a/SIAC/SIAC_L8.py +++ b/SIAC/SIAC_L8.py @@ -15,7 +15,7 @@ home = expanduser("~") file_path = os.path.dirname(os.path.realpath(__file__)) -def SIAC_L8(l8_dir, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/'): +def SIAC_L8(l8_dir, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi = None): file_path = os.path.dirname(os.path.realpath(__file__)) if not os.path.exists(file_path + '/emus/'): os.mkdir(file_path + '/emus/') @@ -35,7 +35,7 @@ def SIAC_L8(l8_dir, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home rets = l8_pre_processing(l8_dir) aero_atmos = [] for ret in rets: - ret += (mcd43, vrt_dir) + ret += (mcd43, vrt_dir, aoi) #sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile = ret aero_atmo = do_correction(*ret) if send_back: @@ -44,7 +44,7 @@ def SIAC_L8(l8_dir, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home return aero_atmos def do_correction(sun_ang_name, view_ang_names, toa_refs, qa_name, cloud_mask, \ - metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/'): + metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi = None): if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'): if not os.path.exists(home + '/MCD43/'): @@ -96,7 +96,7 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, qa_name, cloud_mask, \ atmo = atmospheric_correction(sensor_sat,toa_bands, band_index,view_angles,sun_angles, \ aot = aot, cloud_mask = cloud_mask,tcwv = tcwv, tco3 = tco3, \ aot_unc = aot_unc, tcwv_unc = tcwv_unc, tco3_unc = tco3_unc, \ - rgb = rgb, ref_scale = scale, ref_off = off, emus_dir=file_path+'/emus/') + rgb = rgb, ref_scale = scale, ref_off = off, emus_dir=file_path+'/emus/', aoi=aoi) atmo._doing_correction() return aero, atmo diff --git a/SIAC/SIAC_S2.py b/SIAC/SIAC_S2.py index 3e42254..a9d8ff9 100644 --- a/SIAC/SIAC_S2.py +++ b/SIAC/SIAC_S2.py @@ -14,7 +14,7 @@ home = expanduser("~") file_path = os.path.dirname(os.path.realpath(__file__)) -def SIAC_S2(s2_t, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/'): +def SIAC_S2(s2_t, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi = None): if not os.path.exists(file_path + '/emus/'): os.mkdir(file_path + '/emus/') if len(glob(file_path + '/emus/' + 'isotropic_MSI_emulators_*_x?p_S2?.pkl')) < 12: @@ -31,7 +31,7 @@ def SIAC_S2(s2_t, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home + rets = s2_pre_processing(s2_t) aero_atmos = [] for ret in rets: - ret += (mcd43, vrt_dir) + ret += (mcd43, vrt_dir, aoi) #sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile = ret aero_atmo = do_correction(*ret) if send_back: @@ -40,7 +40,7 @@ def SIAC_S2(s2_t, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home + return aero_atmos def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, \ - cloud_mask, metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/'): + cloud_mask, metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi=None): if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'): if not os.path.exists(home + '/MCD43/'): @@ -70,7 +70,7 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, \ sun_angles = sun_ang_name aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,\ sun_angles,obs_time,cloud_mask, gamma=10., spec_m_dir= \ - file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', mcd43_dir=vrt_dir) + file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', mcd43_dir=vrt_dir, aoi=aoi) aero._solving() toa_bands = toa_refs view_angles = view_ang_names diff --git a/SIAC/tests/aoi.geojson b/SIAC/tests/aoi.geojson new file mode 100644 index 0000000..8c798ca --- /dev/null +++ b/SIAC/tests/aoi.geojson @@ -0,0 +1 @@ +{"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates":[[[0.413360595703125,52.24293795417479],[1.492767333984375,52.24293795417479],[1.492767333984375,52.76870834078517],[0.413360595703125,52.76870834078517],[0.413360595703125,52.24293795417479]]]}}]} \ No newline at end of file diff --git a/SIAC/tests/test_s2.py b/SIAC/tests/test_s2.py index c04370e..8b5fda7 100644 --- a/SIAC/tests/test_s2.py +++ b/SIAC/tests/test_s2.py @@ -39,7 +39,7 @@ def file_as_bytes(file): pass s2_file_dir = filename.split('/')[0] - SIAC_S2(s2_file_dir) + SIAC_S2(s2_file_dir, aoi = myPath + '/aoi.geojson') with open(myPath.replace('tests', '') + 'data/.earthdata_auth', 'wb') as f: f.write(('').encode()) From c83db1480721be0a93e271384da0defc36eac5fa Mon Sep 17 00:00:00 2001 From: MarcYin Date: Tue, 25 Sep 2018 18:05:09 +0100 Subject: [PATCH 060/103] debug... --- SIAC/tests/aoi.geojson | 2 +- SIAC/tests/test_s2.py | 2 +- SIAC/the_aerosol.py | 25 ++++++++++++++++--------- SIAC/the_correction.py | 39 +++++++++++++++++++++++---------------- 4 files changed, 41 insertions(+), 27 deletions(-) diff --git a/SIAC/tests/aoi.geojson b/SIAC/tests/aoi.geojson index 8c798ca..6e95955 100644 --- a/SIAC/tests/aoi.geojson +++ b/SIAC/tests/aoi.geojson @@ -1 +1 @@ -{"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates":[[[0.413360595703125,52.24293795417479],[1.492767333984375,52.24293795417479],[1.492767333984375,52.76870834078517],[0.413360595703125,52.76870834078517],[0.413360595703125,52.24293795417479]]]}}]} \ No newline at end of file +{ "type": "Polygon", "coordinates": [ [ [ 0.413360595703125, 52.242937954174792 ], [ 1.492767333984375, 52.242937954174792 ], [ 1.492767333984375, 52.768708340785167 ], [ 0.413360595703125, 52.768708340785167 ], [ 0.413360595703125, 52.242937954174792 ] ] ] } diff --git a/SIAC/tests/test_s2.py b/SIAC/tests/test_s2.py index 8b5fda7..e773e92 100644 --- a/SIAC/tests/test_s2.py +++ b/SIAC/tests/test_s2.py @@ -39,7 +39,7 @@ def file_as_bytes(file): pass s2_file_dir = filename.split('/')[0] - SIAC_S2(s2_file_dir, aoi = myPath + '/aoi.geojson') + SIAC_S2(s2_file_dir, aoi = str(myPath + '/aoi.geojson')) with open(myPath.replace('tests', '') + 'data/.earthdata_auth', 'wb') as f: f.write(('').encode()) diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index 968f4a2..80ebc09 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -127,6 +127,8 @@ def _create_base_map(self,): ''' Deal with different types way to define the AOI, if none is specified, then the image bound is used. ''' + ogr.UseExceptions() + gdal.UseExceptions() if self.aoi is not None: if os.path.exists(self.aoi): try: @@ -134,27 +136,32 @@ def _create_base_map(self,): subprocess.call(['gdaltindex', '-f', 'GeoJSON', self.toa_dir + '/AOI.json', self.aoi]) except: try: - g = ogr.Open(self.aoi) + gr = ogr.Open(str(self.aoi)) + l = gr.GetLayer(0) + f = l.GetFeature(0) + g = f.GetGeometryRef() except: raise IOError('AOI file cannot be opened by gdal, please check it or transform into format can be opened by gdal') else: try: - g = ogr.CreateGeometryFromJson(aoi) + g = ogr.CreateGeometryFromJson(self.aoi) except: try: - g = ogr.CreateGeometryFromGML(aoi) + g = ogr.CreateGeometryFromGML(self.aoi) except: try: - g = ogr.CreateGeometryFromWkt(aoi) + g = ogr.CreateGeometryFromWkt(self.aoi) except: try: - g = ogr.CreateGeometryFromWkb(aoi) + g = ogr.CreateGeometryFromWkb(self.aoi) except: raise IOError('The AOI has to be one of GeoJSON, GML, Wkt or Wkb.') - gjson_str = '''{"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":%s}]}'''% g.ExportToJson() - - with open(self.toa_dir + '/AOI.json', 'wb') as f: - f.write(gjson_str) + gjson_str = '''{"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":%s}]}'''% g.ExportToJson() + with open(self.toa_dir + '/AOI.json', 'wb') as f: + f.write(gjson_str) + + ogr.DontUseExceptions() + gdal.DontUseExceptions() if not os.path.exists(self.toa_dir + '/AOI.json'): self.logger.warning('AOI is not created and full band extend is used') subprocess.call(['gdaltindex', '-f', 'GeoJSON', self.toa_dir +'/AOI.json', self.toa_bands[0]]) diff --git a/SIAC/the_correction.py b/SIAC/the_correction.py index 50ecdf4..6d06840 100644 --- a/SIAC/the_correction.py +++ b/SIAC/the_correction.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os import sys +import ogr import gdal import psutil import logging @@ -112,34 +113,40 @@ def _create_base_map(self,): ''' Deal with different types way to define the AOI, if none is specified, then the image bound is used. ''' + ogr.UseExceptions() + gdal.UseExceptions() if self.aoi is not None: if os.path.exists(self.aoi): - try: + try: g = gdal.Open(self.aoi) subprocess.call(['gdaltindex', '-f', 'GeoJSON', self.toa_dir + '/AOI.json', self.aoi]) - except: - try: - g = ogr.Open(self.aoi) + except: + try: + gr = ogr.Open(self.aoi) + l = gr.GetLayer(0) + f = l.GetFeature(0) + g = f.GetGeometryRef() except: raise IOError('AOI file cannot be opened by gdal, please check it or transform into format can be opened by gdal') - else: - try: - g = ogr.CreateGeometryFromJson(aoi) - except: - try: - g = ogr.CreateGeometryFromGML(aoi) + else: + try: + g = ogr.CreateGeometryFromJson(self.aoi) + except: + try: + g = ogr.CreateGeometryFromGML(self.aoi) except: try: - g = ogr.CreateGeometryFromWkt(aoi) + g = ogr.CreateGeometryFromWkt(self.aoi) except: try: - g = ogr.CreateGeometryFromWkb(aoi) + g = ogr.CreateGeometryFromWkb(self.aoi) except: raise IOError('The AOI has to be one of GeoJSON, GML, Wkt or Wkb.') - gjson_str = '''{"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":%s}]}'''% g.ExportToJson() - - with open(self.toa_dir + '/AOI.json', 'wb') as f: - f.write(gjson_str) + gjson_str = '''{"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":%s}]}'''% g.ExportToJson() + with open(self.toa_dir + '/AOI.json', 'wb') as f: + f.write(gjson_str) + ogr.DontUseExceptions() + gdal.DontUseExceptions() if not os.path.exists(self.toa_dir + '/AOI.json'): subprocess.call(['gdaltindex', '-f', 'GeoJSON', self.toa_dir +'/AOI.json', self.toa_bands[0]]) self.logger.warning('AOI is not created and full band extend is used') From 5f792bb98ff08e4214d1efb2600fefbfb35ae867 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Tue, 25 Sep 2018 18:48:07 +0100 Subject: [PATCH 061/103] debug... --- SIAC/downloaders.py | 2 +- SIAC/the_aerosol.py | 2 +- SIAC/the_correction.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/SIAC/downloaders.py b/SIAC/downloaders.py index 4421d20..5fc3d78 100644 --- a/SIAC/downloaders.py +++ b/SIAC/downloaders.py @@ -19,7 +19,7 @@ def downloader(fname, url_root, file_dir): new_url = url_root + fname new_req = requests.get(new_url, stream=True) if new_req.ok: - logger.info('downloading %s' % fname) + logger.info('downloading %s and save it at %s' % (fname, os.path.join(file_dir, fname))) with open(os.path.join(file_dir, fname), 'wb') as fp: for chunk in new_req.iter_content(chunk_size=1024): if chunk: diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index 80ebc09..4c50df2 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -158,7 +158,7 @@ def _create_base_map(self,): raise IOError('The AOI has to be one of GeoJSON, GML, Wkt or Wkb.') gjson_str = '''{"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":%s}]}'''% g.ExportToJson() with open(self.toa_dir + '/AOI.json', 'wb') as f: - f.write(gjson_str) + f.write(gjson_str.encode()) ogr.DontUseExceptions() gdal.DontUseExceptions() diff --git a/SIAC/the_correction.py b/SIAC/the_correction.py index 6d06840..1ee7220 100644 --- a/SIAC/the_correction.py +++ b/SIAC/the_correction.py @@ -144,7 +144,7 @@ def _create_base_map(self,): raise IOError('The AOI has to be one of GeoJSON, GML, Wkt or Wkb.') gjson_str = '''{"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":%s}]}'''% g.ExportToJson() with open(self.toa_dir + '/AOI.json', 'wb') as f: - f.write(gjson_str) + f.write(gjson_str.encode()) ogr.DontUseExceptions() gdal.DontUseExceptions() if not os.path.exists(self.toa_dir + '/AOI.json'): From 362c044c511bb6ab5e3764ed930a119f0e3c307e Mon Sep 17 00:00:00 2001 From: MarcYin Date: Tue, 25 Sep 2018 19:39:26 +0100 Subject: [PATCH 062/103] debug --- SIAC/the_aerosol.py | 3 ++- SIAC/the_correction.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index 4c50df2..e80ddce 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -595,7 +595,8 @@ def _load_xa_xb_xc_emus(self,): if sys.version_info >= (3,0): f = lambda em: pkl.load(open(em, 'rb'), encoding = 'latin1') else: - f = lambda em: pkl.load(open(em, 'rb')) + f = lambda em: pkl.load(open(str(em), 'rb')) + print([xap_emu, xbp_emu, xcp_emu]) self.emus = parmap(f, [xap_emu, xbp_emu, xcp_emu]) def _get_convolved_toa(self,): diff --git a/SIAC/the_correction.py b/SIAC/the_correction.py index 1ee7220..8df24a4 100644 --- a/SIAC/the_correction.py +++ b/SIAC/the_correction.py @@ -179,7 +179,8 @@ def _load_xa_xb_xc_emus(self,): if sys.version_info >= (3,0): f = lambda em: pkl.load(open(em, 'rb'), encoding = 'latin1') else: - f = lambda em: pkl.load(open(em, 'rb')) + f = lambda em: pkl.load(open(str(em), 'rb')) + print([xap_emu, xbp_emu, xcp_emu]) self.xap_emus, self.xbp_emus, self.xcp_emus = parmap(f, [xap_emu, xbp_emu, xcp_emu]) def _var_parser(self, var): @@ -388,7 +389,7 @@ def _get_boa(self,): re = parmap(self._do_chunk, bands_to_do, min(int(proc), len(bands_to_do))) else: self._chunks = int(np.ceil(1. / proc)) - re = map(self._do_band) + re = list(map(self._do_chunk, bands_to_do)) ret += re index += (np.where(needed==u_need[i])[0]).tolist() ret = list(zip(*sorted(zip(index, ret))))[1] From 075697ef6120c701fa6c2ed595494fc471ba500e Mon Sep 17 00:00:00 2001 From: MarcYin Date: Tue, 25 Sep 2018 22:31:52 +0100 Subject: [PATCH 063/103] debug --- .travis.yml | 3 ++- SIAC/tests/aoi.geojson | 28 +++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index bf4df5c..bf2cb3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,8 @@ install: - conda info -a - conda list - conda install pip - - pip install gp_emulator==1.6.1 + #- pip install gp_emulator==1.6.1 + - wget https://github.com/MarcYin/gp_emulator/archive/62f288792e1b24a12891fc0ad32896cb7cb5eda4.zip -O gp_emulator.zip;unzip gp_emulator.zip; rm gp_emulator.zip;mv gp_emulator* gp_emulator;cd gp_emulator;python setup.py install;cd ..;rm -rf gp_emulator - pip install coveralls - pip install codecov script: diff --git a/SIAC/tests/aoi.geojson b/SIAC/tests/aoi.geojson index 6e95955..4db855a 100644 --- a/SIAC/tests/aoi.geojson +++ b/SIAC/tests/aoi.geojson @@ -1 +1,27 @@ -{ "type": "Polygon", "coordinates": [ [ [ 0.413360595703125, 52.242937954174792 ], [ 1.492767333984375, 52.242937954174792 ], [ 1.492767333984375, 52.768708340785167 ], [ 0.413360595703125, 52.768708340785167 ], [ 0.413360595703125, 52.242937954174792 ] ] ] } +{ + "type": "Polygon", + "coordinates": [ + [ + [ + 0.5987548828125, + 52.4308969498065 + ], + [ + 1.267547607421875, + 52.4308969498065 + ], + [ + 1.267547607421875, + 52.784492347358785 + ], + [ + 0.5987548828125, + 52.784492347358785 + ], + [ + 0.5987548828125, + 52.4308969498065 + ] + ] + ] + } From 08ea261cff8d646d21ed469e645b3f30aadb59bb Mon Sep 17 00:00:00 2001 From: MarcYin Date: Tue, 25 Sep 2018 23:38:31 +0100 Subject: [PATCH 064/103] further recude aoi --- .travis.yml | 6 ++++-- SIAC/tests/aoi.geojson | 22 +++++++++++----------- SIAC/the_aerosol.py | 2 +- SIAC/the_correction.py | 2 +- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index bf2cb3f..fdad516 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,8 +21,10 @@ install: - conda info -a - conda list - conda install pip - #- pip install gp_emulator==1.6.1 - - wget https://github.com/MarcYin/gp_emulator/archive/62f288792e1b24a12891fc0ad32896cb7cb5eda4.zip -O gp_emulator.zip;unzip gp_emulator.zip; rm gp_emulator.zip;mv gp_emulator* gp_emulator;cd gp_emulator;python setup.py install;cd ..;rm -rf gp_emulator + - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then + wget https://github.com/MarcYin/gp_emulator/archive/62f288792e1b24a12891fc0ad32896cb7cb5eda4.zip -O gp_emulator.zip;unzip gp_emulator.zip; rm gp_emulator.zip;mv gp_emulator* gp_emulator;cd gp_emulator;python setup.py install;cd ..;rm -rf gp_emulator + else + pip install gp_emulator==1.6.1 - pip install coveralls - pip install codecov script: diff --git a/SIAC/tests/aoi.geojson b/SIAC/tests/aoi.geojson index 4db855a..08ab618 100644 --- a/SIAC/tests/aoi.geojson +++ b/SIAC/tests/aoi.geojson @@ -3,25 +3,25 @@ "coordinates": [ [ [ - 0.5987548828125, - 52.4308969498065 + 0.6921386718749999, + 52.51622086393074 ], [ - 1.267547607421875, - 52.4308969498065 + 1.109619140625, + 52.51622086393074 ], [ - 1.267547607421875, - 52.784492347358785 + 1.109619140625, + 52.68304276227741 ], [ - 0.5987548828125, - 52.784492347358785 + 0.6921386718749999, + 52.68304276227741 ], [ - 0.5987548828125, - 52.4308969498065 + 0.6921386718749999, + 52.51622086393074 ] ] ] - } +} diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index e80ddce..5e240f9 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -596,7 +596,7 @@ def _load_xa_xb_xc_emus(self,): f = lambda em: pkl.load(open(em, 'rb'), encoding = 'latin1') else: f = lambda em: pkl.load(open(str(em), 'rb')) - print([xap_emu, xbp_emu, xcp_emu]) + #print([xap_emu, xbp_emu, xcp_emu]) self.emus = parmap(f, [xap_emu, xbp_emu, xcp_emu]) def _get_convolved_toa(self,): diff --git a/SIAC/the_correction.py b/SIAC/the_correction.py index 8df24a4..475604e 100644 --- a/SIAC/the_correction.py +++ b/SIAC/the_correction.py @@ -180,7 +180,7 @@ def _load_xa_xb_xc_emus(self,): f = lambda em: pkl.load(open(em, 'rb'), encoding = 'latin1') else: f = lambda em: pkl.load(open(str(em), 'rb')) - print([xap_emu, xbp_emu, xcp_emu]) + #print([xap_emu, xbp_emu, xcp_emu]) self.xap_emus, self.xbp_emus, self.xcp_emus = parmap(f, [xap_emu, xbp_emu, xcp_emu]) def _var_parser(self, var): From 4b95fece43254e5f75d83162666c6629485d2e33 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 26 Sep 2018 00:08:54 +0100 Subject: [PATCH 065/103] if else --- .travis.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fdad516..e63e5b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,13 @@ install: - conda list - conda install pip - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then - wget https://github.com/MarcYin/gp_emulator/archive/62f288792e1b24a12891fc0ad32896cb7cb5eda4.zip -O gp_emulator.zip;unzip gp_emulator.zip; rm gp_emulator.zip;mv gp_emulator* gp_emulator;cd gp_emulator;python setup.py install;cd ..;rm -rf gp_emulator + wget https://github.com/MarcYin/gp_emulator/archive/62f288792e1b24a12891fc0ad32896cb7cb5eda4.zip -O gp_emulator.zip; + unzip gp_emulator.zip; + rm gp_emulator.zip;mv gp_emulator* gp_emulator; + cd gp_emulator; + python setup.py install; + cd ..; + rm -rf gp_emulator else pip install gp_emulator==1.6.1 - pip install coveralls From 2e10e2a507ed90f013f378b8146c0b2eeacfbd7e Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 26 Sep 2018 00:10:02 +0100 Subject: [PATCH 066/103] if else --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e63e5b6..f8e1be3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,7 @@ install: rm -rf gp_emulator else pip install gp_emulator==1.6.1 + fi - pip install coveralls - pip install codecov script: From cf8de0412681e72ca13d62f4d4b98e099936a717 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 26 Sep 2018 10:20:11 +0100 Subject: [PATCH 067/103] install from link --- .travis.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index f8e1be3..03e1170 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,13 +22,7 @@ install: - conda list - conda install pip - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then - wget https://github.com/MarcYin/gp_emulator/archive/62f288792e1b24a12891fc0ad32896cb7cb5eda4.zip -O gp_emulator.zip; - unzip gp_emulator.zip; - rm gp_emulator.zip;mv gp_emulator* gp_emulator; - cd gp_emulator; - python setup.py install; - cd ..; - rm -rf gp_emulator + pip install https://github.com/MarcYin/gp_emulator/archive/62f288792e1b24a12891fc0ad32896cb7cb5eda4.zip else pip install gp_emulator==1.6.1 fi From 2738bcabfd17cb84de93963f8d0e1d004cde246f Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 26 Sep 2018 11:36:23 +0100 Subject: [PATCH 068/103] emulator --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 03e1170..8e04c4d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,9 @@ install: - conda list - conda install pip - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then - pip install https://github.com/MarcYin/gp_emulator/archive/62f288792e1b24a12891fc0ad32896cb7cb5eda4.zip + pip install https://github.com/MarcYin/gp_emulator/archive/62f288792e1b24a12891fc0ad32896cb7cb5eda4.zip; else - pip install gp_emulator==1.6.1 + pip install gp_emulator==1.6.1; fi - pip install coveralls - pip install codecov From 356a4ff9cccc7caf803feceea3634b33b9ffb456 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 26 Sep 2018 15:09:05 +0100 Subject: [PATCH 069/103] lets pass it --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8e04c4d..16c922d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,4 +39,6 @@ script: after_success: - codecov - coveralls + - rm -rf SIAC/emus/*.pkl + - rm -rf S2?_MSIL1C*.SAFE - test $TRAVIS_BRANCH = "master" && conda install conda-build && conda install anaconda-client && PKG_NAME=siac && USER=f0xy && OS=$TRAVIS_OS_NAME-64 && mkdir ~/conda-bld && conda config --set anaconda_upload no && export CONDA_BLD_PATH=~/conda-bld && export VERSION=`date +%Y.%m.%d` && conda build . -c conda-forge && ls -lah $CONDA_BLD_PATH/$OS && anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $(ls $CONDA_BLD_PATH/$OS/$PKG_NAME-$VERSION*.tar.bz2) --force From 8f0433e9378b6dc523cb8a8bdb2a29262d324d3f Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 26 Sep 2018 16:07:01 +0100 Subject: [PATCH 070/103] fix a bug --- meta.yaml | 9 ++++----- setup.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/meta.yaml b/meta.yaml index ccbef53..59d35a4 100644 --- a/meta.yaml +++ b/meta.yaml @@ -25,6 +25,7 @@ requirements: - requests - scikit-learn - scikit-image + - networkx run: - python - gdal >=2.1,<2.3 @@ -36,13 +37,11 @@ requirements: - requests - scikit-learn - scikit-image + - networkx test: - requires: - - pytest - commands: - - pytest .. - + imports: + - SIAC about: home: https://github.com/MarcYin/SIAC license: GNU Affero General Public License v3.0 diff --git a/setup.py b/setup.py index 8b39f38..89e184c 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ readme = f.read().decode() setup(name ='SIAC', - version ='2.0.7', + version ='2.0.8', description = 'A sensor invariant Atmospheric Correction (SIAC)', long_description = readme, long_description_content_type ='text/markdown', From dc2a9c08ecc50d7eb9dd24ded614c7e7f8007231 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 26 Sep 2018 16:56:46 +0100 Subject: [PATCH 071/103] lets try to pass it --- meta.yaml | 5 ++--- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/meta.yaml b/meta.yaml index 59d35a4..a0edaae 100644 --- a/meta.yaml +++ b/meta.yaml @@ -7,7 +7,7 @@ source: git_tag: master build: - script: pip install gp_emulator==1.6.1 && python setup.py install + script: pip install gp_emulator && python setup.py install script_env: - VERSION - CONDA_BLD_PATH @@ -40,8 +40,7 @@ requirements: - networkx test: - imports: - - SIAC + - about: home: https://github.com/MarcYin/SIAC license: GNU Affero General Public License v3.0 diff --git a/setup.py b/setup.py index 89e184c..fc40d4d 100644 --- a/setup.py +++ b/setup.py @@ -13,8 +13,8 @@ classifiers = ['Development Status :: 4 - Beta', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.6'], - install_requires = ['gdal>=2.1, <2.3', 'numpy>=1.13', 'scipy>=1.0', 'psutil','six', - 'lightgbm>=2.1.0','requests', 'scikit-learn', 'scikit-image', 'gp_emulator==1.6.1'], + install_requires = ['gdal>=2.1, <2.3', 'numpy>=1.13', 'scipy>=1.0', 'psutil','six', 'networkx', + 'lightgbm>=2.1.0','requests', 'scikit-learn', 'scikit-image', 'gp_emulator'], url = 'https://github.com/MarcYin/SIAC', license = "GNU Affero General Public License v3.0", include_package_data = True, From 44b6109b8c2b97f08f71b63f7f016edcd2a4e2a4 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Wed, 26 Sep 2018 23:19:05 +0100 Subject: [PATCH 072/103] add test --- meta.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/meta.yaml b/meta.yaml index a0edaae..5767bfe 100644 --- a/meta.yaml +++ b/meta.yaml @@ -40,7 +40,10 @@ requirements: - networkx test: - - + requires: + - pytest + commands: + - pytest --cov=../SIAC/tests/test_dependency.py about: home: https://github.com/MarcYin/SIAC license: GNU Affero General Public License v3.0 From 58584756e111a2a222ac8ee6e728306b164e55d8 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 27 Sep 2018 09:19:04 +0100 Subject: [PATCH 073/103] fix test --- meta.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/meta.yaml b/meta.yaml index 5767bfe..ca8317d 100644 --- a/meta.yaml +++ b/meta.yaml @@ -40,10 +40,8 @@ requirements: - networkx test: - requires: - - pytest commands: - - pytest --cov=../SIAC/tests/test_dependency.py + - python ../SIAC/tests/test_dependency.py about: home: https://github.com/MarcYin/SIAC license: GNU Affero General Public License v3.0 From dc4cdfe4f4eb35331767987532d79a7fc3cb6c71 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 27 Sep 2018 13:56:01 +0100 Subject: [PATCH 074/103] locallly build works --- meta.yaml | 16 ++++++++-------- setup.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/meta.yaml b/meta.yaml index ca8317d..967f761 100644 --- a/meta.yaml +++ b/meta.yaml @@ -7,10 +7,13 @@ source: git_tag: master build: - script: pip install gp_emulator && python setup.py install - script_env: - - VERSION - - CONDA_BLD_PATH + channels: + - anaconda + - conda-forge + script: pip install networkx && pip install gp_emulator && python setup.py install --single-version-externally-managed --record=record.txt + script_env: + - VERSION + - CONDA_BLD_PATH requirements: host: @@ -25,7 +28,6 @@ requirements: - requests - scikit-learn - scikit-image - - networkx run: - python - gdal >=2.1,<2.3 @@ -37,11 +39,9 @@ requirements: - requests - scikit-learn - scikit-image - - networkx - test: commands: - - python ../SIAC/tests/test_dependency.py + - python SIAC/tests/test_dependency.py about: home: https://github.com/MarcYin/SIAC license: GNU Affero General Public License v3.0 diff --git a/setup.py b/setup.py index fc40d4d..1d417d3 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ classifiers = ['Development Status :: 4 - Beta', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.6'], - install_requires = ['gdal>=2.1, <2.3', 'numpy>=1.13', 'scipy>=1.0', 'psutil','six', 'networkx', + install_requires = ['gdal>=2.1, <2.3', 'numpy>=1.13', 'scipy>=1.0', 'psutil','six', 'lightgbm>=2.1.0','requests', 'scikit-learn', 'scikit-image', 'gp_emulator'], url = 'https://github.com/MarcYin/SIAC', license = "GNU Affero General Public License v3.0", From e42a12169b9f504355f477c7d04958702ab10826 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Thu, 27 Sep 2018 14:54:36 +0100 Subject: [PATCH 075/103] hellow world --- meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta.yaml b/meta.yaml index 967f761..0fb0d62 100644 --- a/meta.yaml +++ b/meta.yaml @@ -41,7 +41,7 @@ requirements: - scikit-image test: commands: - - python SIAC/tests/test_dependency.py + - echo 'Hello world' about: home: https://github.com/MarcYin/SIAC license: GNU Affero General Public License v3.0 From 363b3037d84f981b7c0854fb4ac3cb9698066d9b Mon Sep 17 00:00:00 2001 From: MarcYin Date: Tue, 2 Oct 2018 14:32:34 +0100 Subject: [PATCH 076/103] debug for band split --- SIAC/atmo_solver.py | 27 ++++++++++++++++++++++++--- SIAC/s2_angle.py | 4 ++-- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/SIAC/atmo_solver.py b/SIAC/atmo_solver.py index a77589a..9e13f66 100644 --- a/SIAC/atmo_solver.py +++ b/SIAC/atmo_solver.py @@ -10,7 +10,7 @@ import cPickle as pkl except: import pickle as pkl -from scipy import optimize, interpolate +from scipy import optimize, interpolate, sparse #from fastDiff import fastDiff from SIAC.multi_process import parmap @@ -29,6 +29,23 @@ class bcolors: BOLD = '\033[1m' UNDERLINE = '\033[4m' +def compose_dtd(nx, ny): + ns = nx*ny + n = int(np.sqrt(ns)) + d1 = 2 * np.ones(ns) + d1[ny-1::ny] = 1 + d1[0::ny] = 1 + d2 = np.ones(ns) * -1 + d2[ny-1::ny] = 0 + d3 = 2 * np.ones(ns) + d3[:ny] = 1 + d3[ns-ny:] = 1 + d4 = np.ones(ns) * -1 + dtdx = sparse.spdiags([d1, d2[::-1], d2], [0, 1, -1], ns, ns) + dtdy = sparse.spdiags([d3, d4, d4], [0, ny, -ny], ns, ns) + dtd = dtdx + dtdy + return dtd, dtdx, dtdy + class solving_atmo_paras(object): ''' A simple implementation of dark dense vegitation method for the restieval of prior aot. @@ -223,8 +240,12 @@ def _multi_grid_solver(self,): #self.tcwv_unc [mask] = self.prior_uncs[1].reshape(shape)[mask] else: self._obs_cost_test(psolve['x'], do_unc = True) - unc = (np.nansum([self.obs_unc.reshape(2, -1), 1. / self.prior_uncs**2 + self.gamma**2], axis = 0)) ** -0.5 - self.aot_unc, self.tcwv_unc = unc.reshape(2, self.num_blocks_x, self.num_blocks_y) + nx, ny = self.prior_uncs.shape + dtd = compose_dtd(nx, ny)[0].todense() + to_inv = np.nansum([np.diag((self.obs_unc).ravel()), np.diag((self.prior_uncs**-2).ravel()), self.gamma**2 * dtd], axis = 0) + unc = np.diag(np.linalg.inv(to_inv))** 0.5 + #unc = (np.nansum([self.obs_unc.reshape(nx, -1), self.prior_uncs**-2 , self.gamma**2], axis = 0)) ** -0.5 + self.aot_unc, self.tcwv_unc = unc.reshape(nx, self.num_blocks_x, self.num_blocks_y) self.tco3_prior, self.tco3_unc = self._grid_conversion(self.tco3_prior, shape), self._grid_conversion(self.tco3_unc, shape) post_solved = np.array([self.aot_prior, self.tcwv_prior, self.tco3_prior]) post_unc = np.array([self.aot_unc, self.tcwv_unc, self.tco3_unc]) diff --git a/SIAC/s2_angle.py b/SIAC/s2_angle.py index 0e5bbea..dcae28d 100644 --- a/SIAC/s2_angle.py +++ b/SIAC/s2_angle.py @@ -236,14 +236,14 @@ def resample_s2_angles(metafile): gmls = glob(s2_file_dir + '/QI_DATA/*MSK_DETFOO*.gml') sun_ang_name = s2_file_dir + '/ANG_DATA/' + 'SAA_SZA.tif' view_ang_names = [s2_file_dir + '/ANG_DATA/' + 'VAA_VZA_%s.tif'%band for band in bands] - toa_refs = glob(s2_file_dir + '/IMG_DATA/*B*.jp2') + toa_refs = glob(s2_file_dir + '/IMG_DATA/*B??.jp2') elif 'metadata.xml' in metafile: if not os.path.exists(s2_file_dir + '/angles/'): os.mkdir(s2_file_dir + '/angles/') gmls = glob(s2_file_dir + '/qi/*MSK_DETFOO*.gml') sun_ang_name = s2_file_dir + '/angles/' + 'SAA_SZA.tif' view_ang_names = [s2_file_dir + '/angles/' + 'VAA_VZA_%s.tif'%band for band in bands] - toa_refs = glob(s2_file_dir + '/*B*.jp2') + toa_refs = glob(s2_file_dir + '/*B??.jp2') else: raise IOError('Invalid metafile please use the default AWS or SCIHUB format.') gmls = sorted(gmls, key = lambda gml: bands.index('B' + gml.split('B')[-1][:2])) From c511b020c3bf2fb6c1807b83232c2aa144ca2ab1 Mon Sep 17 00:00:00 2001 From: MarcYin Date: Fri, 5 Oct 2018 16:43:39 +0100 Subject: [PATCH 077/103] uncertainty... --- SIAC/atmo_solver.py | 14 +++++++++++--- setup.py | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/SIAC/atmo_solver.py b/SIAC/atmo_solver.py index 9e13f66..a5407f8 100644 --- a/SIAC/atmo_solver.py +++ b/SIAC/atmo_solver.py @@ -11,6 +11,7 @@ except: import pickle as pkl from scipy import optimize, interpolate, sparse +from scipy.sparse import linalg #from fastDiff import fastDiff from SIAC.multi_process import parmap @@ -241,9 +242,16 @@ def _multi_grid_solver(self,): else: self._obs_cost_test(psolve['x'], do_unc = True) nx, ny = self.prior_uncs.shape - dtd = compose_dtd(nx, ny)[0].todense() - to_inv = np.nansum([np.diag((self.obs_unc).ravel()), np.diag((self.prior_uncs**-2).ravel()), self.gamma**2 * dtd], axis = 0) - unc = np.diag(np.linalg.inv(to_inv))** 0.5 + dtd = compose_dtd(1, ny)[0] + self.obs_unc[np.isnan(self.obs_unc)] = 0 + #self.prior_uncs[np.isnan(self.prior_uncs)] = np.mean(self.prior_uncs[~np.isnan(self.prior_uncs)]) + to_inv = np.nansum([sparse.diags((self.obs_unc[0]).ravel()), sparse.diags((self.prior_uncs[0]**-2).ravel()), self.gamma**2 * dtd], axis = 0) + aot_unc = (linalg.inv(to_inv).diagonal())** 0.5 + + to_inv = np.nansum([sparse.diags((self.obs_unc[1]).ravel()), sparse.diags((self.prior_uncs[1]**-2).ravel()), self.gamma**2 * dtd], axis = 0) + tcwv_unc = (linalg.inv(to_inv).diagonal())** 0.5 + + unc = np.array([aot_unc, tcwv_unc]) #unc = (np.nansum([self.obs_unc.reshape(nx, -1), self.prior_uncs**-2 , self.gamma**2], axis = 0)) ** -0.5 self.aot_unc, self.tcwv_unc = unc.reshape(nx, self.num_blocks_x, self.num_blocks_y) self.tco3_prior, self.tco3_unc = self._grid_conversion(self.tco3_prior, shape), self._grid_conversion(self.tco3_unc, shape) diff --git a/setup.py b/setup.py index 1d417d3..8c2afb4 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ readme = f.read().decode() setup(name ='SIAC', - version ='2.0.8', + version ='2.0.9', description = 'A sensor invariant Atmospheric Correction (SIAC)', long_description = readme, long_description_content_type ='text/markdown', From 26b0aec5a79e14f74e30999f27e03a7fcb4219ab Mon Sep 17 00:00:00 2001 From: TonioF Date: Thu, 1 Nov 2018 16:29:23 +0100 Subject: [PATCH 078/103] moved authentication --- SIAC/get_MCD43.py | 50 +++++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/SIAC/get_MCD43.py b/SIAC/get_MCD43.py index d2f4328..d7e76b6 100644 --- a/SIAC/get_MCD43.py +++ b/SIAC/get_MCD43.py @@ -18,30 +18,6 @@ file_path = os.path.dirname(os.path.realpath(__file__)) logger = create_logger() - -if os.path.exists(file_path + '/data/.earthdata_auth'): - try: - username, password = np.loadtxt(file_path + '/data/.earthdata_auth', dtype=str) - auth = tuple([username, password]) - except: - logger.error('Please provide NASA Earthdata username and password for downloading MCD43 data, which can be applied here: https://urs.earthdata.nasa.gov.') - username = input('Username for NASA Earthdata: ') - password = getpass.getpass('Password for NASA Earthdata: ') - up = username, password - os.remove(file_path + '/data/.earthdata_auth') - with open(file_path + '/data/.earthdata_auth', 'wb') as f: - for i in up: - f.write((i + '\n').encode()) - auth = tuple([username, password]) -else: - username = input('Username for NASA Earthdata: ') - password = getpass.getpass('Password for NASA Earthdata: ') - up = username, password - with open(home + '/.earthdata_auth', 'wb') as f: - for i in up: - f.write((i + '\n').encode()) - auth = tuple([username, password]) - #print(file_path + '/data/.earthdata_auth') #print(auth) @@ -76,6 +52,28 @@ def get_one_tile(tile_date): return base + date + '/' + fname[0] def downloader(url_fname): + if os.path.exists(file_path + '/data/.earthdata_auth'): + try: + username, password = np.loadtxt(file_path + '/data/.earthdata_auth', dtype=str) + auth = tuple([username, password]) + except: + logger.error('Please provide NASA Earthdata username and password for downloading MCD43 data, which can be applied here: https://urs.earthdata.nasa.gov.') + username = input('Username for NASA Earthdata: ') + password = getpass.getpass('Password for NASA Earthdata: ') + up = username, password + os.remove(file_path + '/data/.earthdata_auth') + with open(file_path + '/data/.earthdata_auth', 'wb') as f: + for i in up: + f.write((i + '\n').encode()) + auth = tuple([username, password]) + else: + username = input('Username for NASA Earthdata: ') + password = getpass.getpass('Password for NASA Earthdata: ') + up = username, password + with open(home + '/.earthdata_auth', 'wb') as f: + for i in up: + f.write((i + '\n').encode()) + auth = tuple([username, password]) url, fname = url_fname with requests.Session() as s: s.max_redirects = 100000 @@ -156,7 +154,3 @@ def get_mcd43(aoi, obs_time, mcd43_dir = './MCD43/', vrt_dir = './MCD43_VRT/'): aoi = '/home/ucfafyi/DATA/S2_MODIS/l_data/LC08_L1TP_014034_20170831_20170915_01_T1/aot.tif' obs_time = datetime(2017, 7, 8, 10, 8, 20) ret = get_mcd43(aoi, obs_time, mcd43_dir = '/home/ucfafyi/hep/MCD43/', vrt_dir = '/home/ucfafyi/DATA/Multiply/MCD43/') - - - - From 69fe19175cce5fc2e06791476df015f11fa4fd4d Mon Sep 17 00:00:00 2001 From: TonioF Date: Thu, 1 Nov 2018 16:30:01 +0100 Subject: [PATCH 079/103] allow to pass in existing files --- SIAC/SIAC_S2.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/SIAC/SIAC_S2.py b/SIAC/SIAC_S2.py index a9d8ff9..7776231 100644 --- a/SIAC/SIAC_S2.py +++ b/SIAC/SIAC_S2.py @@ -1,5 +1,5 @@ +import argparse import os -import sys import requests import numpy as np from glob import glob @@ -10,14 +10,20 @@ from SIAC.s2_preprocessing import s2_pre_processing from SIAC.downloaders import downloader from SIAC.multi_process import parmap +from SIAC.create_logger import create_logger from os.path import expanduser home = expanduser("~") file_path = os.path.dirname(os.path.realpath(__file__)) -def SIAC_S2(s2_t, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi = None): - if not os.path.exists(file_path + '/emus/'): - os.mkdir(file_path + '/emus/') - if len(glob(file_path + '/emus/' + 'isotropic_MSI_emulators_*_x?p_S2?.pkl')) < 12: +logger = create_logger() + +def SIAC_S2(s2_t, dem_vrt, cams_dir, send_back = False, mcd43 = home + '/MCD43/', emu_dir = file_path + '/emus/', + vrt_dir = home + '/MCD43_VRT/', aoi = None): + logger.info('Starting SIAC') + if not os.path.exists(emu_dir): + emu_dir = file_path + '/emus/' + os.mkdir(emu_dir) + if len(glob(emu_dir + '/' + 'isotropic_MSI_emulators_*_x?p_S2?.pkl')) < 12: url = 'http://www2.geog.ucl.ac.uk/~ucfafyi/emus/' req = requests.get(url) to_down = [] @@ -26,12 +32,12 @@ def SIAC_S2(s2_t, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home + fname = line.split('"')[1].split('<')[0] if 'MSI' in fname: to_down.append([fname, url]) - f = lambda fname_url: downloader(fname_url[0], fname_url[1], file_path + '/emus/') + f = lambda fname_url: downloader(fname_url[0], fname_url[1], emu_dir) parmap(f, to_down) rets = s2_pre_processing(s2_t) aero_atmos = [] for ret in rets: - ret += (mcd43, vrt_dir, aoi) + ret += (dem_vrt, cams_dir, mcd43, vrt_dir, aoi) #sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile = ret aero_atmo = do_correction(*ret) if send_back: @@ -39,8 +45,8 @@ def SIAC_S2(s2_t, send_back = False, mcd43 = home + '/MCD43/', vrt_dir = home + if send_back: return aero_atmos -def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, \ - cloud_mask, metafile, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi=None): +def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile, + dem_vrt, cams_dir, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi=None): if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'): if not os.path.exists(home + '/MCD43/'): @@ -70,7 +76,8 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, \ sun_angles = sun_ang_name aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,\ sun_angles,obs_time,cloud_mask, gamma=10., spec_m_dir= \ - file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', mcd43_dir=vrt_dir, aoi=aoi) + file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', mcd43_dir=vrt_dir, aoi=aoi, + global_dem=dem_vrt, cams_dir=cams_dir) aero._solving() toa_bands = toa_refs view_angles = view_ang_names @@ -90,3 +97,13 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, \ atmo._doing_correction() return aero, atmo +parser = argparse.ArgumentParser(description='Sentinel 2 Atmospheric Correction Excutable') +parser.add_argument('-f', "--file_path", help='Sentinel 2 file path in the form of AWS', required=True) +parser.add_argument("-m", "--MCD43_file_dir", help="Directory where you store MCD43A1.006 data") +parser.add_argument("-e", "--emulator_dir", help="Directory where you store emulators.") +parser.add_argument("-d", "--dem", help="A global dem file, and a vrt file is recommended.") +# parser.add_argument("-w", "--wv_emulator", help="A water vapour restrieval emulator.") +parser.add_argument("-c", "--cams", help="Directory where you store cams data.") +parser.add_argument("-a", "--aoi", help="Area of Interest.") +args = parser.parse_args() +SIAC_S2(s2_t=args.file_path, dem_vrt=args.dem, cams_dir=args.cams, mcd43=args.MCD43_file_dir, emu_dir=args.emulator_dir, aoi=args.aoi) From 64320a1d3f00e400b6e68c70a3d6bb5cd032402f Mon Sep 17 00:00:00 2001 From: TonioF Date: Thu, 1 Nov 2018 16:30:15 +0100 Subject: [PATCH 080/103] added environment.yml --- environment.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 environment.yml diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..e051e3c --- /dev/null +++ b/environment.yml @@ -0,0 +1,16 @@ +name: siac-dev + +channels: + - conda-forge + +dependencies: + - python=3.6* + - gdal=2.2 + - numpy>=1.13 + - scipy=1.0 + - psutil + - six + - lightgbm=2.1.0 + - requests + - scikit-learn + - scikit-image From 6509ff954b7c769b90c448ecc26e2490e1174cf0 Mon Sep 17 00:00:00 2001 From: TonioF Date: Mon, 5 Nov 2018 17:32:58 +0100 Subject: [PATCH 081/103] added logger --- SIAC/s2_angle.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/SIAC/s2_angle.py b/SIAC/s2_angle.py index dcae28d..77729e4 100644 --- a/SIAC/s2_angle.py +++ b/SIAC/s2_angle.py @@ -12,6 +12,18 @@ from multiprocessing import Pool import xml.etree.ElementTree as ET #from scipy.interpolate import griddata +from SIAC.create_logger import create_logger +''' +logger = logging.getLogger('SIAC') +logger.setLevel(logging.INFO) +if not logger.handlers: + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) +''' +logger = create_logger() def parse_xml(meta_file, example_file, sun_ang_name): tree = ET.parse(meta_file) @@ -83,6 +95,7 @@ def parse_xml(meta_file, example_file, sun_ang_name): def get_angle(view_ang_name_gml, vaa, vza, band_dict): band_name, view_ang_name, gml = view_ang_name_gml + logger.info('getting angle {}' % (view_ang_name_gml)) g = ogr.Open(gml) xRes = 10; yRes=10 g1 = gdal.Open(band_name) From 5f616c939dc315fe6665d5c01fc9f0fe9f60cefd Mon Sep 17 00:00:00 2001 From: TonioF Date: Mon, 5 Nov 2018 17:42:22 +0100 Subject: [PATCH 082/103] added logger --- SIAC/s2_angle.py | 1 + 1 file changed, 1 insertion(+) diff --git a/SIAC/s2_angle.py b/SIAC/s2_angle.py index 77729e4..653d33b 100644 --- a/SIAC/s2_angle.py +++ b/SIAC/s2_angle.py @@ -266,6 +266,7 @@ def resample_s2_angles(metafile): vaa, vza = parse_xml(metafile, example_file, sun_ang_name) view_ang_name_gmls = zip(toa_refs, view_ang_names, gmls) band_dict = dict(zip(bands, range(13))) + logger.info('getting angles from {} processes'.format(procs)) par = partial(get_angle, vaa=vaa, vza=vza, band_dict=band_dict) p = Pool(procs) ret = p.map(par, view_ang_name_gmls) From a5d4c621c64fd236659a49b2cc37ce2a388b77ad Mon Sep 17 00:00:00 2001 From: TonioF Date: Tue, 6 Nov 2018 11:30:29 +0100 Subject: [PATCH 083/103] do not retrieve modis data --- SIAC/SIAC_S2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SIAC/SIAC_S2.py b/SIAC/SIAC_S2.py index 7776231..a7e9e91 100644 --- a/SIAC/SIAC_S2.py +++ b/SIAC/SIAC_S2.py @@ -65,8 +65,8 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask obs_time = datetime.strptime(sensing_time, u'%Y-%m-%dT%H:%M:%S.%fZ') if 'TILE_ID' in i: sat = i.split('')[-1].split('_')[0] - - get_mcd43(toa_refs[0], obs_time, mcd43_dir = mcd43, vrt_dir = vrt_dir) + + # get_mcd43(toa_refs[0], obs_time, mcd43_dir = mcd43, vrt_dir = vrt_dir) #get_mcd43(toa_refs[0], obs_time, mcd43_dir = '/home/ucfafyi/hep/MCD43/', vrt_dir = '/home/ucfafyi/DATA/Multiply/MCD43/') sensor_sat = 'MSI', sat band_index = [1,2,3,7,11,12] From e9880a5a18be00831a5ce93e23ee2092660ba16f Mon Sep 17 00:00:00 2001 From: TonioF Date: Tue, 6 Nov 2018 11:30:42 +0100 Subject: [PATCH 084/103] do not use multiprocessing --- SIAC/s2_angle.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/SIAC/s2_angle.py b/SIAC/s2_angle.py index 653d33b..9a74fba 100644 --- a/SIAC/s2_angle.py +++ b/SIAC/s2_angle.py @@ -95,7 +95,7 @@ def parse_xml(meta_file, example_file, sun_ang_name): def get_angle(view_ang_name_gml, vaa, vza, band_dict): band_name, view_ang_name, gml = view_ang_name_gml - logger.info('getting angle {}' % (view_ang_name_gml)) + logger.info('getting angle for {}, {}, and {}'.format(band_name, view_ang_name, gml)) g = ogr.Open(gml) xRes = 10; yRes=10 g1 = gdal.Open(band_name) @@ -267,15 +267,14 @@ def resample_s2_angles(metafile): view_ang_name_gmls = zip(toa_refs, view_ang_names, gmls) band_dict = dict(zip(bands, range(13))) logger.info('getting angles from {} processes'.format(procs)) - par = partial(get_angle, vaa=vaa, vza=vza, band_dict=band_dict) - p = Pool(procs) - ret = p.map(par, view_ang_name_gmls) - p.close() - p.join() - ret = np.array(ret) view_ang_names = np.array(view_ang_names) - bad_angs = view_ang_names[~ret] - src_files = view_ang_names[ret][abs(np.arange(13)[~ret][...,None] - np.arange(13)[ret]).argmin(axis=1)] + ret = [True] * 13 + for i, view_ang_name_gml in enumerate(view_ang_name_gmls): + got_angle = get_angle(view_ang_name_gml, vaa, vza, band_dict) + ret[i] = got_angle + inv_ret = [not i for i in ret] + bad_angs = view_ang_names[inv_ret] + src_files = view_ang_names[ret][abs(np.arange(13)[inv_ret][...,None] - np.arange(13)[ret]).argmin(axis=1)] for i in range(len(bad_angs)): copyfile(src_files[i], bad_angs[i]) return sun_ang_name, view_ang_names, toa_refs, cloud_name From 4e9ab39661d6c6d734d4a17164fb79c37b95d8bd Mon Sep 17 00:00:00 2001 From: TonioF Date: Tue, 6 Nov 2018 16:43:41 +0100 Subject: [PATCH 085/103] added switch for modis data retrieval --- SIAC/SIAC_S2.py | 35 ++++++++++++++++++++---------- SIAC/get_MCD43.py | 54 ++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 73 insertions(+), 16 deletions(-) diff --git a/SIAC/SIAC_S2.py b/SIAC/SIAC_S2.py index a7e9e91..9e718d4 100644 --- a/SIAC/SIAC_S2.py +++ b/SIAC/SIAC_S2.py @@ -3,7 +3,7 @@ import requests import numpy as np from glob import glob -from SIAC.get_MCD43 import get_mcd43 +from SIAC.get_MCD43 import get_mcd43, get_locaL_MCD43 from datetime import datetime from SIAC.the_aerosol import solve_aerosol from SIAC.the_correction import atmospheric_correction @@ -17,9 +17,20 @@ logger = create_logger() + +def _ensure_dir_format(dir: str): + if not dir.endswith('/'): + dir += '/' + return dir + + def SIAC_S2(s2_t, dem_vrt, cams_dir, send_back = False, mcd43 = home + '/MCD43/', emu_dir = file_path + '/emus/', - vrt_dir = home + '/MCD43_VRT/', aoi = None): + vrt_dir = home + '/MCD43_VRT/', download_mcd43 = 'True', aoi = None): logger.info('Starting SIAC') + cams_dir = _ensure_dir_format(cams_dir) + mcd43 = _ensure_dir_format(mcd43) + emu_dir = _ensure_dir_format(emu_dir) + vrt_dir = _ensure_dir_format(vrt_dir) if not os.path.exists(emu_dir): emu_dir = file_path + '/emus/' os.mkdir(emu_dir) @@ -34,10 +45,11 @@ def SIAC_S2(s2_t, dem_vrt, cams_dir, send_back = False, mcd43 = home + '/MCD43/' to_down.append([fname, url]) f = lambda fname_url: downloader(fname_url[0], fname_url[1], emu_dir) parmap(f, to_down) + download_mcd43 = download_mcd43 == 'True' rets = s2_pre_processing(s2_t) aero_atmos = [] for ret in rets: - ret += (dem_vrt, cams_dir, mcd43, vrt_dir, aoi) + ret += (dem_vrt, cams_dir, mcd43, vrt_dir, download_mcd43, aoi) #sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile = ret aero_atmo = do_correction(*ret) if send_back: @@ -45,8 +57,8 @@ def SIAC_S2(s2_t, dem_vrt, cams_dir, send_back = False, mcd43 = home + '/MCD43/' if send_back: return aero_atmos -def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile, - dem_vrt, cams_dir, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', aoi=None): +def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile, dem_vrt, cams_dir, + mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', download_mcd_43 = True, aoi=None): if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'): if not os.path.exists(home + '/MCD43/'): @@ -56,7 +68,6 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask if not os.path.exists(home + '/MCD43_VRT/'): os.mkdir(home + '/MCD43_VRT/') - base = os.path.dirname(toa_refs[0]) base = toa_refs[0].replace('B01.jp2', '') with open(metafile) as f: for i in f.readlines(): @@ -65,9 +76,10 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask obs_time = datetime.strptime(sensing_time, u'%Y-%m-%dT%H:%M:%S.%fZ') if 'TILE_ID' in i: sat = i.split('')[-1].split('_')[0] - - # get_mcd43(toa_refs[0], obs_time, mcd43_dir = mcd43, vrt_dir = vrt_dir) - #get_mcd43(toa_refs[0], obs_time, mcd43_dir = '/home/ucfafyi/hep/MCD43/', vrt_dir = '/home/ucfafyi/DATA/Multiply/MCD43/') + if download_mcd_43: + get_mcd43(toa_refs[0], obs_time, mcd43_dir = mcd43, vrt_dir = vrt_dir) + else: + get_locaL_MCD43(toa_refs[0], obs_time, mcd43_dir=mcd43, vrt_dir=vrt_dir) sensor_sat = 'MSI', sat band_index = [1,2,3,7,11,12] band_wv = [469, 555, 645, 859, 1640, 2130] @@ -102,8 +114,9 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask parser.add_argument("-m", "--MCD43_file_dir", help="Directory where you store MCD43A1.006 data") parser.add_argument("-e", "--emulator_dir", help="Directory where you store emulators.") parser.add_argument("-d", "--dem", help="A global dem file, and a vrt file is recommended.") -# parser.add_argument("-w", "--wv_emulator", help="A water vapour restrieval emulator.") +parser.add_argument("-o", "--download", help="Whether to download MCD 43 Data.") parser.add_argument("-c", "--cams", help="Directory where you store cams data.") parser.add_argument("-a", "--aoi", help="Area of Interest.") args = parser.parse_args() -SIAC_S2(s2_t=args.file_path, dem_vrt=args.dem, cams_dir=args.cams, mcd43=args.MCD43_file_dir, emu_dir=args.emulator_dir, aoi=args.aoi) +SIAC_S2(s2_t=args.file_path, dem_vrt=args.dem, cams_dir=args.cams, mcd43=args.MCD43_file_dir, + vrt_dir=args.MCD43_file_dir, download_mcd43=args.download, emu_dir=args.emulator_dir, aoi=args.aoi) diff --git a/SIAC/get_MCD43.py b/SIAC/get_MCD43.py index d7e76b6..d42f229 100644 --- a/SIAC/get_MCD43.py +++ b/SIAC/get_MCD43.py @@ -21,6 +21,26 @@ #print(file_path + '/data/.earthdata_auth') #print(auth) + +def find_local_files(aoi, obs_time, temporal_window = 16): + days = [(obs_time - timedelta(days = int(i))).timetuple() for i in np.arange(temporal_window, 0, -1)] + \ + [(obs_time + timedelta(days = int(i))).timetuple() for i in np.arange(0, temporal_window+1, 1)] + try: + tiles = get_vector_hv(aoi) + except: + try: + tiles = get_raster_hv(aoi) + except: + raise IOError('AOI has to be raster or vector object/files.') + mcd43_file_pattern = 'MCD43A1.A{}{:03}.{}.006.*.hdf' + expected_files = [] + for day in days: + for tile in tiles: + expected_files.append(mcd43_file_pattern.format(day.tm_year, day.tm_yday, tile)) + return expected_files + + + def find_files(aoi, obs_time, temporal_window = 16): days = [(obs_time - timedelta(days = int(i))).strftime('%Y.%m.%d') for i in np.arange(temporal_window, 0, -1)] + \ [(obs_time + timedelta(days = int(i))).strftime('%Y.%m.%d') for i in np.arange(0, temporal_window+1, 1)] @@ -38,6 +58,7 @@ def find_files(aoi, obs_time, temporal_window = 16): p.join() return ret + def get_one_tile(tile_date): base = 'https://e4ftl01.cr.usgs.gov/MOTA/MCD43A1.006/' tile, date = tile_date @@ -51,6 +72,7 @@ def get_one_tile(tile_date): time.sleep(1) return base + date + '/' + fname[0] + def downloader(url_fname): if os.path.exists(file_path + '/data/.earthdata_auth'): try: @@ -101,7 +123,8 @@ def downloader(url_fname): raise IOError('Failed to download the whole file.') else: print(r.content) - + + def daily_vrt(fnames_date, vrt_dir = None): temp1 = 'HDF4_EOS:EOS_GRID:"%s":MOD_Grid_BRDF:BRDF_Albedo_Band_Mandatory_Quality_%s' temp2 = 'HDF4_EOS:EOS_GRID:"%s":MOD_Grid_BRDF:BRDF_Albedo_Parameters_%s' @@ -124,8 +147,28 @@ def daily_vrt(fnames_date, vrt_dir = None): for band in ['Band1','Band2','Band3','Band4','Band5','Band6','Band7', 'vis', 'nir', 'shortwave']: bs = [] for fname in all_files: - bs.append(temp%(fname, band)) - gdal.BuildVRT(date_dir + '_'.join(['MCD43', date, bs[0].split(':')[-1]])+'.vrt', bs).FlushCache() + bs.append(temp%(fname, band)) + vrt_data_set = gdal.BuildVRT(date_dir + '_'.join(['MCD43', date, bs[0].split(':')[-1]])+'.vrt', bs) + if vrt_data_set is None: + logger.info('Could not create vrt') + else: + vrt_data_set.FlushCache() + + +def get_locaL_MCD43(aoi, obs_time, mcd43_dir = './MCD43/', vrt_dir = './MCD43_VRT/'): + logger.propagate = False + expected_files = find_local_files(aoi, obs_time, 16) + files = [] + for file in expected_files: + files.append(glob(mcd43_dir + '/' + file)) + flist = np.array(files).flatten() + all_dates = np.array([i.split('/')[-1] .split('.')[1][1:9] for i in flist]) + udates = np.unique(all_dates) + fnames_dates = [[flist[all_dates==date].tolist(),date] for date in udates] + for fname_date in fnames_dates: + logger.info('Creating daily VRT...') + daily_vrt(fname_date, vrt_dir) + def get_mcd43(aoi, obs_time, mcd43_dir = './MCD43/', vrt_dir = './MCD43_VRT/'): logger.propagate = False @@ -144,11 +187,12 @@ def get_mcd43(aoi, obs_time, mcd43_dir = './MCD43/', vrt_dir = './MCD43_VRT/'): fnames_dates = [[flist[all_dates==date].tolist(),date] for date in udates] logger.info('Creating daily VRT...') par = partial(daily_vrt, vrt_dir = vrt_dir) - p = Pool(len(fnames_dates)) + p = Pool(len(fnames_dates)) p.map(par, fnames_dates) - p.close() + p.close() p.join() + if __name__ == '__main__': aoi = '/home/ucfafyi/DATA/S2_MODIS/l_data/LC08_L1TP_014034_20170831_20170915_01_T1/AOI.json' aoi = '/home/ucfafyi/DATA/S2_MODIS/l_data/LC08_L1TP_014034_20170831_20170915_01_T1/aot.tif' From e0fc103c278bf010859ecee46721c80e10c31cc6 Mon Sep 17 00:00:00 2001 From: TonioF Date: Tue, 6 Nov 2018 16:52:08 +0100 Subject: [PATCH 086/103] typo --- SIAC/the_aerosol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index 5e240f9..be3267c 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -668,7 +668,7 @@ def _solving(self,): self._resamplers() self.logger.info('Parsing angles.') self._parse_angles() - self.logger.info('Mask bad pixeles.') + self.logger.info('Mask bad pixels.') self._mask_bad_pix() self.logger.info('Get simulated BOA.') self._get_boa() From ad999cd5740a1d1021aced601240875413f9c3c5 Mon Sep 17 00:00:00 2001 From: TonioF Date: Tue, 6 Nov 2018 18:04:22 +0100 Subject: [PATCH 087/103] use passed emulator dir --- SIAC/SIAC_S2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SIAC/SIAC_S2.py b/SIAC/SIAC_S2.py index 9e718d4..d938e66 100644 --- a/SIAC/SIAC_S2.py +++ b/SIAC/SIAC_S2.py @@ -49,7 +49,7 @@ def SIAC_S2(s2_t, dem_vrt, cams_dir, send_back = False, mcd43 = home + '/MCD43/' rets = s2_pre_processing(s2_t) aero_atmos = [] for ret in rets: - ret += (dem_vrt, cams_dir, mcd43, vrt_dir, download_mcd43, aoi) + ret += (dem_vrt, cams_dir, emu_dir, mcd43, vrt_dir, download_mcd43, aoi) #sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile = ret aero_atmo = do_correction(*ret) if send_back: @@ -58,7 +58,7 @@ def SIAC_S2(s2_t, dem_vrt, cams_dir, send_back = False, mcd43 = home + '/MCD43/' return aero_atmos def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile, dem_vrt, cams_dir, - mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', download_mcd_43 = True, aoi=None): + emus_dir, mcd43 = home + '/MCD43/', vrt_dir = home + '/MCD43_VRT/', download_mcd_43 = True, aoi=None): if os.path.realpath(mcd43) in os.path.realpath(home + '/MCD43/'): if not os.path.exists(home + '/MCD43/'): @@ -88,7 +88,7 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask sun_angles = sun_ang_name aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,\ sun_angles,obs_time,cloud_mask, gamma=10., spec_m_dir= \ - file_path+'/spectral_mapping/', emus_dir=file_path+'/emus/', mcd43_dir=vrt_dir, aoi=aoi, + file_path+'/spectral_mapping/', emus_dir=emus_dir, mcd43_dir=vrt_dir, aoi=aoi, global_dem=dem_vrt, cams_dir=cams_dir) aero._solving() toa_bands = toa_refs From 7a55c9146d531508f007f7108c9f51e5bd5ea941 Mon Sep 17 00:00:00 2001 From: TonioF Date: Wed, 7 Nov 2018 16:15:39 +0100 Subject: [PATCH 088/103] pass correct directories --- SIAC/SIAC_S2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SIAC/SIAC_S2.py b/SIAC/SIAC_S2.py index d938e66..9d6abf4 100644 --- a/SIAC/SIAC_S2.py +++ b/SIAC/SIAC_S2.py @@ -105,7 +105,7 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask sun_angles, aot = aot, cloud_mask = cloud_mask,\ tcwv = tcwv, tco3 = tco3, aot_unc = aot_unc, \ tcwv_unc = tcwv_unc, tco3_unc = tco3_unc, rgb = \ - rgb, emus_dir=file_path+'/emus/') + rgb, emus_dir=emus_dir, global_dem=dem_vrt, cams_dir=cams_dir) atmo._doing_correction() return aero, atmo From 5d1200bc3f46c1ff0076a838099785040cd9ee2f Mon Sep 17 00:00:00 2001 From: TonioF Date: Wed, 7 Nov 2018 19:50:41 +0100 Subject: [PATCH 089/103] use main --- SIAC/SIAC_S2.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/SIAC/SIAC_S2.py b/SIAC/SIAC_S2.py index 9d6abf4..9a61908 100644 --- a/SIAC/SIAC_S2.py +++ b/SIAC/SIAC_S2.py @@ -108,15 +108,15 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask rgb, emus_dir=emus_dir, global_dem=dem_vrt, cams_dir=cams_dir) atmo._doing_correction() return aero, atmo - -parser = argparse.ArgumentParser(description='Sentinel 2 Atmospheric Correction Excutable') -parser.add_argument('-f', "--file_path", help='Sentinel 2 file path in the form of AWS', required=True) -parser.add_argument("-m", "--MCD43_file_dir", help="Directory where you store MCD43A1.006 data") -parser.add_argument("-e", "--emulator_dir", help="Directory where you store emulators.") -parser.add_argument("-d", "--dem", help="A global dem file, and a vrt file is recommended.") -parser.add_argument("-o", "--download", help="Whether to download MCD 43 Data.") -parser.add_argument("-c", "--cams", help="Directory where you store cams data.") -parser.add_argument("-a", "--aoi", help="Area of Interest.") -args = parser.parse_args() -SIAC_S2(s2_t=args.file_path, dem_vrt=args.dem, cams_dir=args.cams, mcd43=args.MCD43_file_dir, +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Sentinel 2 Atmospheric Correction Excutable') + parser.add_argument('-f', "--file_path", help='Sentinel 2 file path in the form of AWS', required=True) + parser.add_argument("-m", "--MCD43_file_dir", help="Directory where you store MCD43A1.006 data") + parser.add_argument("-e", "--emulator_dir", help="Directory where you store emulators.") + parser.add_argument("-d", "--dem", help="A global dem file, and a vrt file is recommended.") + parser.add_argument("-o", "--download", help="Whether to download MCD 43 Data.") + parser.add_argument("-c", "--cams", help="Directory where you store cams data.") + parser.add_argument("-a", "--aoi", help="Area of Interest.") + args = parser.parse_args() + SIAC_S2(s2_t=args.file_path, dem_vrt=args.dem, cams_dir=args.cams, mcd43=args.MCD43_file_dir, vrt_dir=args.MCD43_file_dir, download_mcd43=args.download, emu_dir=args.emulator_dir, aoi=args.aoi) From d51aa7c497cefbce6f5a8203d5bb43dddcc81c21 Mon Sep 17 00:00:00 2001 From: TonioF Date: Tue, 27 Nov 2018 18:17:45 +0100 Subject: [PATCH 090/103] added logger for debugging --- SIAC/SIAC_S2.py | 4 ++-- SIAC/get_MCD43.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/SIAC/SIAC_S2.py b/SIAC/SIAC_S2.py index 9a61908..0fc1452 100644 --- a/SIAC/SIAC_S2.py +++ b/SIAC/SIAC_S2.py @@ -3,7 +3,7 @@ import requests import numpy as np from glob import glob -from SIAC.get_MCD43 import get_mcd43, get_locaL_MCD43 +from SIAC.get_MCD43 import get_mcd43, get_local_MCD43 from datetime import datetime from SIAC.the_aerosol import solve_aerosol from SIAC.the_correction import atmospheric_correction @@ -79,7 +79,7 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask if download_mcd_43: get_mcd43(toa_refs[0], obs_time, mcd43_dir = mcd43, vrt_dir = vrt_dir) else: - get_locaL_MCD43(toa_refs[0], obs_time, mcd43_dir=mcd43, vrt_dir=vrt_dir) + get_local_MCD43(toa_refs[0], obs_time, mcd43_dir=mcd43, vrt_dir=vrt_dir) sensor_sat = 'MSI', sat band_index = [1,2,3,7,11,12] band_wv = [469, 555, 645, 859, 1640, 2130] diff --git a/SIAC/get_MCD43.py b/SIAC/get_MCD43.py index d42f229..adb86a7 100644 --- a/SIAC/get_MCD43.py +++ b/SIAC/get_MCD43.py @@ -155,12 +155,14 @@ def daily_vrt(fnames_date, vrt_dir = None): vrt_data_set.FlushCache() -def get_locaL_MCD43(aoi, obs_time, mcd43_dir = './MCD43/', vrt_dir = './MCD43_VRT/'): +def get_local_MCD43(aoi, obs_time, mcd43_dir ='./MCD43/', vrt_dir ='./MCD43_VRT/'): logger.propagate = False expected_files = find_local_files(aoi, obs_time, 16) files = [] for file in expected_files: - files.append(glob(mcd43_dir + '/' + file)) + globbed_file = mcd43_dir + '/' + file + logger.info('globbed file: {}'.format(globbed_file)) + files.append(glob(globbed_file)) flist = np.array(files).flatten() all_dates = np.array([i.split('/')[-1] .split('.')[1][1:9] for i in flist]) udates = np.unique(all_dates) From 8d8d2bfa8fd552f3ee77b3ef7cafba2a0ada4ba8 Mon Sep 17 00:00:00 2001 From: TonioF Date: Tue, 27 Nov 2018 18:30:15 +0100 Subject: [PATCH 091/103] fix --- SIAC/get_MCD43.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SIAC/get_MCD43.py b/SIAC/get_MCD43.py index adb86a7..e748527 100644 --- a/SIAC/get_MCD43.py +++ b/SIAC/get_MCD43.py @@ -160,9 +160,9 @@ def get_local_MCD43(aoi, obs_time, mcd43_dir ='./MCD43/', vrt_dir ='./MCD43_VRT/ expected_files = find_local_files(aoi, obs_time, 16) files = [] for file in expected_files: - globbed_file = mcd43_dir + '/' + file + globbed_file = glob(mcd43_dir + '/' + file) logger.info('globbed file: {}'.format(globbed_file)) - files.append(glob(globbed_file)) + files.append(globbed_file) flist = np.array(files).flatten() all_dates = np.array([i.split('/')[-1] .split('.')[1][1:9] for i in flist]) udates = np.unique(all_dates) From a7a9e06b1a771f473a88f2637a6d74a7105faafb Mon Sep 17 00:00:00 2001 From: TonioF Date: Tue, 27 Nov 2018 18:40:52 +0100 Subject: [PATCH 092/103] fix --- SIAC/get_MCD43.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/SIAC/get_MCD43.py b/SIAC/get_MCD43.py index e748527..5c9ca3f 100644 --- a/SIAC/get_MCD43.py +++ b/SIAC/get_MCD43.py @@ -160,9 +160,11 @@ def get_local_MCD43(aoi, obs_time, mcd43_dir ='./MCD43/', vrt_dir ='./MCD43_VRT/ expected_files = find_local_files(aoi, obs_time, 16) files = [] for file in expected_files: - globbed_file = glob(mcd43_dir + '/' + file) - logger.info('globbed file: {}'.format(globbed_file)) - files.append(globbed_file) + globbed_files = glob(mcd43_dir + '/' + file) + for globbed_file in globbed_files: + if len(globbed_file) > 0: + files.append(globbed_file) + logger.info('globbed file: {}'.format(globbed_file)) flist = np.array(files).flatten() all_dates = np.array([i.split('/')[-1] .split('.')[1][1:9] for i in flist]) udates = np.unique(all_dates) From cbfd384a7f5e76ffd357fdc26851231ba79a715a Mon Sep 17 00:00:00 2001 From: TonioF Date: Thu, 29 Nov 2018 10:21:24 +0100 Subject: [PATCH 093/103] added logs --- SIAC/the_aerosol.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index be3267c..cce3693 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -642,8 +642,11 @@ def _re_mask(self,): def _fill_nan(self,): def fill_nan(array): - x_shp, y_shp = array.shape - mask = ~np.isnan(array) + x_shp, y_shp = array.shape + logging.info('x_shp {}'.format(x_shp)) + logging.info('y_shp {}'.format(y_shp)) + logging.info('array {}'.format(array)) + mask = ~np.isnan(array) valid = np.array(np.where(mask)).T value = array[mask] mesh = np.repeat(range(x_shp), y_shp).reshape(x_shp, y_shp), \ From 736ccb9d4bc4be33b7d7cb1d6f080e365b2edc9e Mon Sep 17 00:00:00 2001 From: TonioF Date: Thu, 29 Nov 2018 10:36:14 +0100 Subject: [PATCH 094/103] print --- SIAC/the_aerosol.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index cce3693..5c24dc2 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -643,9 +643,9 @@ def _re_mask(self,): def _fill_nan(self,): def fill_nan(array): x_shp, y_shp = array.shape - logging.info('x_shp {}'.format(x_shp)) - logging.info('y_shp {}'.format(y_shp)) - logging.info('array {}'.format(array)) + print('x_shp {}'.format(x_shp)) + print('y_shp {}'.format(y_shp)) + print('array {}'.format(array)) mask = ~np.isnan(array) valid = np.array(np.where(mask)).T value = array[mask] From f6a2ff09d8b4b5598ce1e5238d5e546e6e62549d Mon Sep 17 00:00:00 2001 From: TonioF Date: Thu, 29 Nov 2018 11:55:48 +0100 Subject: [PATCH 095/103] removed annoying prints/logs --- SIAC/get_MCD43.py | 2 -- SIAC/the_aerosol.py | 3 --- 2 files changed, 5 deletions(-) diff --git a/SIAC/get_MCD43.py b/SIAC/get_MCD43.py index 5c9ca3f..e90fe04 100644 --- a/SIAC/get_MCD43.py +++ b/SIAC/get_MCD43.py @@ -164,13 +164,11 @@ def get_local_MCD43(aoi, obs_time, mcd43_dir ='./MCD43/', vrt_dir ='./MCD43_VRT/ for globbed_file in globbed_files: if len(globbed_file) > 0: files.append(globbed_file) - logger.info('globbed file: {}'.format(globbed_file)) flist = np.array(files).flatten() all_dates = np.array([i.split('/')[-1] .split('.')[1][1:9] for i in flist]) udates = np.unique(all_dates) fnames_dates = [[flist[all_dates==date].tolist(),date] for date in udates] for fname_date in fnames_dates: - logger.info('Creating daily VRT...') daily_vrt(fname_date, vrt_dir) diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index 5c24dc2..4def201 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -643,9 +643,6 @@ def _re_mask(self,): def _fill_nan(self,): def fill_nan(array): x_shp, y_shp = array.shape - print('x_shp {}'.format(x_shp)) - print('y_shp {}'.format(y_shp)) - print('array {}'.format(array)) mask = ~np.isnan(array) valid = np.array(np.where(mask)).T value = array[mask] From 536b37988c8cb234ed8ec8bd34d4afdcc611af15 Mon Sep 17 00:00:00 2001 From: TonioF Date: Wed, 16 Jan 2019 11:05:11 +0100 Subject: [PATCH 096/103] commented out lines to deal with notebook halt issue --- SIAC/the_correction.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/SIAC/the_correction.py b/SIAC/the_correction.py index 475604e..830d8ea 100644 --- a/SIAC/the_correction.py +++ b/SIAC/the_correction.py @@ -515,19 +515,19 @@ def _compose_rgb(self,): projection = self._toa_bands[self.ri].GetProjectionRef() geotransform = self._toa_bands[self.ri].GetGeoTransform() self._save_rgb(rgba_array, name, projection, geotransform) - gdal.Translate(self.toa_dir +'/TOA_overview.png', self.toa_dir+'/TOA_RGB.tif', \ - format = 'PNG', widthPct=25, heightPct=25, resampleAlg=gdal.GRA_Bilinear ).FlushCache() - gdal.Translate(self.toa_dir +'/TOA_ovr.png', self.toa_dir+'/TOA_RGB.tif', \ - format = 'PNG', widthPct=10, heightPct=10, resampleAlg=gdal.GRA_Bilinear ).FlushCache() + # gdal.Translate(self.toa_dir +'/TOA_overview.png', self.toa_dir+'/TOA_RGB.tif', \ + # format = 'PNG', widthPct=25, heightPct=25, resampleAlg=gdal.GRA_Bilinear ).FlushCache() + # gdal.Translate(self.toa_dir +'/TOA_ovr.png', self.toa_dir+'/TOA_RGB.tif', \ + # format = 'PNG', widthPct=10, heightPct=10, resampleAlg=gdal.GRA_Bilinear ).FlushCache() rgba_array = np.clip([self.boa_rgb[0] * self.rgb_scale * 255, self.boa_rgb[1] * self.rgb_scale * 255, \ self.boa_rgb[2] * self.rgb_scale * 255, alpha * self.rgb_scale * 255], 0, 255).astype(np.uint8) name = self.toa_dir + '/BOA_RGB.tif' self._save_rgb(rgba_array, name, projection, geotransform) - gdal.Translate(self.toa_dir+'/BOA_overview.png', self.toa_dir+'/BOA_RGB.tif', \ - format = 'PNG', widthPct=25, heightPct=25, resampleAlg=gdal.GRA_Bilinear ).FlushCache() - gdal.Translate(self.toa_dir+'/BOA_ovr.png', self.toa_dir+'/BOA_RGB.tif', \ - format = 'PNG', widthPct=10, heightPct=10, resampleAlg=gdal.GRA_Bilinear ).FlushCache() + # gdal.Translate(self.toa_dir+'/BOA_overview.png', self.toa_dir+'/BOA_RGB.tif', \ + # format = 'PNG', widthPct=25, heightPct=25, resampleAlg=gdal.GRA_Bilinear ).FlushCache() + # gdal.Translate(self.toa_dir+'/BOA_ovr.png', self.toa_dir+'/BOA_RGB.tif', \ + # format = 'PNG', widthPct=10, heightPct=10, resampleAlg=gdal.GRA_Bilinear ).FlushCache() def _doing_correction(self,): self.logger.propagate = False From 9e807e323d1982afa7d7522379d93c2566bba3de Mon Sep 17 00:00:00 2001 From: TonioF Date: Tue, 9 Apr 2019 17:51:13 +0200 Subject: [PATCH 097/103] added logger --- SIAC/get_MCD43.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/SIAC/get_MCD43.py b/SIAC/get_MCD43.py index f9aa2ca..f0cf6d6 100644 --- a/SIAC/get_MCD43.py +++ b/SIAC/get_MCD43.py @@ -2,22 +2,20 @@ import time import gdal import getpass -import logging import requests import numpy as np from glob import glob from six.moves import input from functools import partial -from os.path import expanduser from multiprocessing import Pool from datetime import datetime, timedelta -from SIAC.modis_tile_cal import get_vector_hv, get_raster_hv from os.path import expanduser from SIAC.create_logger import create_logger from SIAC.modis_tile_cal import get_vector_hv, get_raster_hv home = expanduser("~") file_path = os.path.dirname(os.path.realpath(__file__)) +logger = create_logger() test_url = 'https://e4ftl01.cr.usgs.gov/MOTA/MCD43A1.006/2000.02.24/MCD43A1.A2000055.h32v08.006.2016101152216.hdf.xml' def get_auth(logger): From d3dfc31c6b551ca718309b767581559797b8d1de Mon Sep 17 00:00:00 2001 From: TonioF Date: Tue, 9 Apr 2019 17:56:59 +0200 Subject: [PATCH 098/103] corrected indentation --- SIAC/SIAC_S2.py | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/SIAC/SIAC_S2.py b/SIAC/SIAC_S2.py index 88915ee..0a78ad4 100755 --- a/SIAC/SIAC_S2.py +++ b/SIAC/SIAC_S2.py @@ -80,25 +80,21 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask if 'TILE_ID' in i: sat = i.split('')[-1].split('_')[0] tile = i.split('')[-1] - log_file = os.path.dirname(metafile) + '/SIAC_S2.log' - logger = create_logger(log_file) - logger.info('Starting atmospheric corretion for %s' % tile) - if not np.all(cloud_mask): - handlers = logger.handlers[:] - for handler in handlers: - handler.close() - logger.removeHandler(handler) - if download_mcd_43: - get_mcd43(toa_refs[0], obs_time, mcd43_dir=mcd43, vrt_dir=vrt_dir) - else: - get_local_MCD43(toa_refs[0], obs_time, mcd43_dir=mcd43, vrt_dir=vrt_dir) - # logger = create_logger(log_file) - else: - logger.info('No clean pixel in this scene and no MCD43 is downloaded.') - sat = i.split('')[-1].split('_')[0] - - get_mcd43(toa_refs[0], obs_time, mcd43_dir = mcd43, vrt_dir = vrt_dir, log_file = log_file) - #get_mcd43(toa_refs[0], obs_time, mcd43_dir = '/home/ucfafyi/hep/MCD43/', vrt_dir = '/home/ucfafyi/DATA/Multiply/MCD43/') + log_file = os.path.dirname(metafile) + '/SIAC_S2.log' + logger = create_logger(log_file) + logger.info('Starting atmospheric corretion for %s' % tile) + if not np.all(cloud_mask): + handlers = logger.handlers[:] + for handler in handlers: + handler.close() + logger.removeHandler(handler) + if download_mcd_43: + get_mcd43(toa_refs[0], obs_time, mcd43_dir=mcd43, vrt_dir=vrt_dir, log_file=log_file) + else: + get_local_MCD43(toa_refs[0], obs_time, mcd43_dir=mcd43, vrt_dir=vrt_dir) + # logger = create_logger(log_file) + else: + logger.info('No clean pixel in this scene and no MCD43 is downloaded.') sensor_sat = 'MSI', sat band_index = [1,2,3,7,11,12] band_wv = [469, 555, 645, 859, 1640, 2130] From a9b06b7f234600b75889805c03d2f484429e32f9 Mon Sep 17 00:00:00 2001 From: TonioF Date: Wed, 10 Apr 2019 10:41:13 +0200 Subject: [PATCH 099/103] consider case when all data is invalid --- SIAC/the_aerosol.py | 2 ++ SIAC/the_correction.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index ed103c6..3b1c627 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -651,6 +651,8 @@ def _fill_nan(self,): def fill_nan(array): x_shp, y_shp = array.shape mask = ~np.isnan(array) + if np.sum(mask) == 0: # no valid data + return np.full((x_shp, y_shp), np.nan) valid = np.array(np.where(mask)).T value = array[mask] mesh = np.repeat(range(x_shp), y_shp).reshape(x_shp, y_shp), \ diff --git a/SIAC/the_correction.py b/SIAC/the_correction.py index c30ba54..0b579cc 100644 --- a/SIAC/the_correction.py +++ b/SIAC/the_correction.py @@ -326,8 +326,10 @@ def _parse_aux(self,): def _fill_nan(self,): def fill_nan(array): - x_shp, y_shp = array.shape - mask = ~np.isnan(array) + x_shp, y_shp = array.shape + mask = ~np.isnan(array) + if np.sum(mask) == 0: # all data is invalid + return np.full((x_shp, y_shp), np.nan) valid = np.array(np.where(mask)).T value = array[mask] mesh = np.repeat(range(x_shp), y_shp).reshape(x_shp, y_shp), \ From 6883188598b6a7933e3e0b3a4fd59dcc900a8940 Mon Sep 17 00:00:00 2001 From: TonioF Date: Wed, 10 Apr 2019 10:53:28 +0200 Subject: [PATCH 100/103] still do not use multiprocessing --- SIAC/s2_angle.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/SIAC/s2_angle.py b/SIAC/s2_angle.py index d3c9a92..41df764 100644 --- a/SIAC/s2_angle.py +++ b/SIAC/s2_angle.py @@ -288,18 +288,18 @@ def resample_s2_angles(metafile): src_files = view_ang_names[ret][abs(np.arange(13)[inv_ret][...,None] - np.arange(13)[ret]).argmin(axis=1)] for i in range(len(bad_angs)): copyfile(src_files[i], bad_angs[i]) - par = partial(get_angle, vaa=vaa, vza=vza, band_dict=band_dict) - p = Pool(procs) - ret = p.map(par, view_ang_name_gmls) + #par = partial(get_angle, vaa=vaa, vza=vza, band_dict=band_dict) + #p = Pool(procs) + #ret = p.map(par, view_ang_name_gmls) #ret =list( map(par, view_ang_name_gmls)) - p.close() - p.join() - ret = np.array(ret) - if ret.sum()>0: - bad_angs = view_ang_names[~ret] - src_files = view_ang_names[ret][abs(np.arange(13)[~ret][...,None] - np.arange(13)[ret]).argmin(axis=1)] - for i in range(len(bad_angs)): - copyfile(src_files[i], bad_angs[i]) - else: - raise LookupError('failed to reconstract angles...') + #p.close() + #p.join() + #ret = np.array(ret) + #if ret.sum()>0: + # bad_angs = view_ang_names[~ret] + # src_files = view_ang_names[ret][abs(np.arange(13)[~ret][...,None] - np.arange(13)[ret]).argmin(axis=1)] + # for i in range(len(bad_angs)): + # copyfile(src_files[i], bad_angs[i]) + #else: + # raise LookupError('failed to reconstract angles...') return sun_ang_name, view_ang_names, toa_refs, cloud_name From 6fe216fe610ad02079cd04af720bdf75ca0c3f24 Mon Sep 17 00:00:00 2001 From: TonioF Date: Wed, 10 Apr 2019 10:41:13 +0200 Subject: [PATCH 101/103] consider case when all data is invalid (cherry picked from commit a9b06b7) --- SIAC/the_aerosol.py | 233 +++++++++++++++++++---------------------- SIAC/the_correction.py | 81 ++++++-------- 2 files changed, 138 insertions(+), 176 deletions(-) diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index 3b1c627..4def201 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -4,11 +4,13 @@ import os import gc import sys +import ogr +import osr +import gdal import SIAC.kernels as kernels import logging -import platform import warnings -warnings.filterwarnings("ignore") +import platform import subprocess import numpy as np from glob import glob @@ -16,21 +18,22 @@ import cPickle as pkl except: import pickle as pkl -from functools import partial -from osgeo import ogr, osr, gdal +#from osgeo import ogr, osr, gdal from SIAC.smoothn import smoothn -from scipy.fftpack import dct, idct +from functools import partial from SIAC.multi_process import parmap +from scipy.fftpack import dct, idct +from SIAC.psf_optimize import psf_optimize from scipy.interpolate import griddata from datetime import datetime, timedelta -from SIAC.psf_optimize import psf_optimize from SIAC.create_logger import create_logger -from SIAC.raster_boundary import get_boundary from SIAC.atmo_solver import solving_atmo_paras from sklearn.linear_model import HuberRegressor from SIAC.reproject import reproject_data, array_to_raster from scipy.ndimage import binary_dilation, binary_erosion +warnings.filterwarnings("ignore") + class solve_aerosol(object): ''' Prepareing modis data to be able to pass into @@ -55,16 +58,17 @@ def __init__(self, aot_unc = None, wv_unc = None, o3_unc = None, - log_file = None, ref_scale = 0.0001, ref_off = 0, ang_scale = 0.01, ele_scale = 0.001, prior_scale = [1., 0.1, 46.698, 1., 1., 1.], emus_dir = 'SIAC/emus/', - mcd43_dir = '~/DATA/Multiply/MCD43/', + mcd43_dir = '/home/ucfafyi/DATA/Multiply/MCD43/', global_dem = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt', cams_dir = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/', + #global_dem = '/home/ucfafyi/DATA/Multiply/eles/global_dem.vrt', + #cams_dir = '/home/ucfafyi/netapp_10/cams/', spec_m_dir = 'SIAC/spectral_mapping', aero_res = 1000): self.sensor = sensor_sat[0] @@ -86,7 +90,6 @@ def __init__(self, self.aot_unc = aot_unc self.wv_unc = wv_unc self.o3_unc = o3_unc - self.log_file = log_file self.ref_scale = ref_scale self.ref_off = ref_off self.ang_scale = ang_scale @@ -101,27 +104,36 @@ def __init__(self, self.mcd43_tmp = '%s/MCD43A1.A%d%03d.%s.006.*.hdf' self.toa_dir = os.path.abspath('/'.join(toa_bands[0].split('/')[:-1])) try: - spec_map = np.loadtxt(spec_m_dir + '/TERRA_%s_spectral_mapping.txt'%self.satellite).T + spec_map = np.loadtxt(spec_m_dir + '/TERRA_%s_spectral_mapping.txt'%self.satellite).T except: - spec_map = np.loadtxt(spec_m_dir + '/TERRA_%s_spectral_mapping.txt'%self.sensor).T + spec_map = np.loadtxt(spec_m_dir + '/TERRA_%s_spectral_mapping.txt'%self.sensor).T self.spec_slope = spec_map[0] self.spec_off = spec_map[1] - self.logger = create_logger(self.log_file) + ''' + # create logger + self.logger = logging.getLogger('SIAC') + self.logger.setLevel(logging.INFO) + if not self.logger.handlers: + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + ch.setFormatter(formatter) + self.logger.addHandler(ch) + self.logger.propagate = False + ''' + self.logger = create_logger() def _create_base_map(self,): ''' Deal with different types way to define the AOI, if none is specified, then the image bound is used. ''' - gdal.UseExceptions() ogr.UseExceptions() + gdal.UseExceptions() if self.aoi is not None: if os.path.exists(self.aoi): try: g = gdal.Open(self.aoi) - #subprocess.call(['gdaltindex', '-f', 'GeoJSON', '-t_srs', 'EPSG:4326', self.toa_dir + '/AOI.json', self.aoi]) - geojson = get_boundary(self.aoi)[0] - with open(self.toa_dir + '/AOI.json', 'wb') as f: - f.write(geojson.encode()) + subprocess.call(['gdaltindex', '-f', 'GeoJSON', self.toa_dir + '/AOI.json', self.aoi]) except: try: gr = ogr.Open(str(self.aoi)) @@ -151,20 +163,8 @@ def _create_base_map(self,): ogr.DontUseExceptions() gdal.DontUseExceptions() if not os.path.exists(self.toa_dir + '/AOI.json'): - g = gdal.Open(self.toa_bands[0]) - proj = g.GetProjection() - if 'WGS 84' in proj: - #subprocess.call(['gdaltindex', '-f', 'GeoJSON', self.toa_dir +'/AOI.json', self.toa_bands[0]]) - geojson = get_boundary(self.toa_bands[0], to_wgs84 = False) - with open(self.toa_dir + '/AOI.json', 'wb') as f: - f.write(geojson.encode()) - else: - #subprocess.call(['gdaltindex', '-f', 'GeoJSON', '-t_srs', 'EPSG:4326', self.toa_dir +'/AOI.json', self.toa_bands[0]]) - geojson = get_boundary(self.toa_bands[0])[0] - with open(self.toa_dir + '/AOI.json', 'wb') as f: - f.write(geojson.encode()) - self.logger.warning('AOI is not created and full band extend is used') + subprocess.call(['gdaltindex', '-f', 'GeoJSON', self.toa_dir +'/AOI.json', self.toa_bands[0]]) self.aoi = self.toa_dir + '/AOI.json' else: self.aoi = self.toa_dir + '/AOI.json' @@ -259,7 +259,7 @@ def _mask_bad_pix(self): mask = binary_erosion (mask, structure = np.ones((3,3)).astype(bool), iterations=5).astype(bool) #mask = binary_dilation(mask, structure = np.ones((3,3)).astype(bool), iterations=30).astype(bool) mask = binary_dilation(mask, structure = np.ones((3,3)).astype(bool), iterations=30+ker_size).astype(bool) - mask[:30,:] = mask[:,:30] = mask[:,-30:] = mask[-30:,:] = True + #mask[:30,:] = mask[:,:30] = mask[:,-30:] = mask[-30:,:] = True self.bad_pix = mask def _create_band_gs(self,): @@ -418,9 +418,9 @@ def _get_bounds(self,): def _read_MCD43(self,fnames): def warp_data(fname, aoi, xRes, yRes): - g = gdal.Warp('',fname, format = 'MEM', srcNodata = 32767, dstNodata=0, \ - cutlineDSName=aoi, xRes = xRes, yRes = yRes, cropToCutline=True, resampleAlg = 0) # weird adaptation for gdal 2.3, this should be a bug in gdal 2.3 - return g.ReadAsArray() # no reason you have to specify the srcNodata to use dstNodata + g = gdal.Warp('',fname, format = 'MEM', dstNodata=0, cutlineDSName=aoi, xRes = \ + xRes, yRes = yRes, cropToCutline=True, resampleAlg = 0) + return g.ReadAsArray() par = partial(warp_data, aoi = self.aoi, xRes = self.aero_res*0.5, yRes = self.aero_res*0.5) n_files = int(len(fnames)/2) ret = parmap(par, fnames) @@ -506,18 +506,10 @@ def _annoying_angles(self, destination): _sun_angles = [] _view_angles = [] for fname in self._sun_angles: - try: - nodatas = ' '.join([i.split("=")[1] for i in gdal.Info(fname).split('\n') if' NoData' in i]) - except: - nodatas = None ang = reproject_data(fname, mg, srcNodata = None, resample = \ 0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data _sun_angles.append(ang) for fname in self._view_angles: - try: - nodatas = ' '.join([i.split("=")[1] for i in gdal.Info(fname).split('\n') if' NoData' in i]) - except: - nodatas = None ang = reproject_data(fname, mg, srcNodata = None, resample = \ 0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data _view_angles.append(ang) @@ -576,6 +568,25 @@ def _get_index(self, mg, hg): hy = hy[rmask] hx = hx[rmask] return hx, hy, hmask, rmask + ''' + def _load_xa_xb_xc_emus(self,): + #Find needed emulators based on the sensor and satellite names. + + xap_emu = glob(self.emus_dir + '/isotropic_%s_emulators_optimization_xap_%s.pkl'%(self.sensor, self.satellite))[0] + xbp_emu = glob(self.emus_dir + '/isotropic_%s_emulators_optimization_xbp_%s.pkl'%(self.sensor, self.satellite))[0] + xcp_emu = glob(self.emus_dir + '/isotropic_%s_emulators_optimization_xcp_%s.pkl'%(self.sensor, self.satellite))[0] + def read_emus(fname): + if sys.version_info >= (3,0): + #f = open(fname, mode='rb', encoding = 'latin1') + f = lambda em: pkl.load(open(em, 'rb'), encoding = 'latin1') + else: + #f = open(fname, mode='rb') + f = lambda em: pkl.load(open(em, 'rb')) + return pkl.load(f) + gc.disable() + self.emus = parmap(read_emus, [xap_emu, xbp_emu, xcp_emu]) + gc.enable() + ''' def _load_xa_xb_xc_emus(self,): xap_emu = glob(self.emus_dir + '/isotropic_%s_emulators_correction_xap_%s.pkl'%(self.sensor, self.satellite))[0] @@ -585,36 +596,17 @@ def _load_xa_xb_xc_emus(self,): f = lambda em: pkl.load(open(em, 'rb'), encoding = 'latin1') else: f = lambda em: pkl.load(open(str(em), 'rb')) + #print([xap_emu, xbp_emu, xcp_emu]) self.emus = parmap(f, [xap_emu, xbp_emu, xcp_emu]) - def _pad_even_shape(self, array): - x_size, y_size = array.shape - if x_size % 2 != 0: - array = np.insert(array, -1, array[-1, :], axis=0) - if y_size % 2 != 0: - array = np.insert(array, -1, array[:, -1], axis=1) - return array - - def _get_convolved_toa(self,): imgs = [band_g.ReadAsArray() for band_g in self._toa_bands] self.bad_pixs = self.bad_pix[self.hx, self.hy] - if self.full_res[0] %2 != 0: - xgaus = np.exp(-2.*(np.pi**2)*(self.psf_xstd**2)*((0.5 * np.arange(self.full_res[0] + 1) /(self.full_res[0] + 1))**2)) - else: - xgaus = np.exp(-2.*(np.pi**2)*(self.psf_xstd**2)*((0.5 * np.arange(self.full_res[0]) /self.full_res[0])**2)) - if self.full_res[1] %2 != 0: - ygaus = np.exp(-2.*(np.pi**2)*(self.psf_ystd**2)*((0.5 * np.arange(self.full_res[1] + 1) /(self.full_res[1] + 1))**2)) - else: - ygaus = np.exp(-2.*(np.pi**2)*(self.psf_ystd**2)*((0.5 * np.arange(self.full_res[1]) /self.full_res[1])**2)) + xgaus = np.exp(-2.*(np.pi**2)*(self.psf_xstd**2)*((0.5 * np.arange(self.full_res[0]) /self.full_res[0])**2)) + ygaus = np.exp(-2.*(np.pi**2)*(self.psf_ystd**2)*((0.5 * np.arange(self.full_res[1]) /self.full_res[1])**2)) gaus_2d = np.outer(xgaus, ygaus) def convolve(img, gaus_2d, hx, hy): - x_size, y_size = img.shape - if x_size % 2 != 0: - img = np.insert(img, -1, img[-1, :], axis=0) - if y_size % 2 != 0: - img = np.insert(img, -1, img[:, -1], axis=1) dat = idct(idct(dct(dct(img, axis=0, norm = 'ortho'), axis=1, \ norm='ortho') * gaus_2d, axis=1, norm='ortho'), axis=0, norm='ortho')[hx, hy] return dat @@ -625,6 +617,7 @@ def convolve(img, gaus_2d, hx, hy): self.ref_off = self.ref_off[self.hx, self.hy] self.toa = np.array(parmap(par,imgs)) * self.ref_scale+self.ref_off + def _re_mask(self,): boa_mask = np.all(self.boa >= 0.001, axis = 0) &\ np.all(self.boa < 1, axis = 0) @@ -635,7 +628,7 @@ def _re_mask(self,): self.toa = self.toa [:, _mask] self.boa = self.boa [:, _mask] * self.spec_slope[...,None] + self.spec_off[...,None] self.boa_unc = self.boa_unc[:, _mask] - eps=1.35 + eps=1.7 mask = True if self.boa.shape[1] > 3: for i in range(len(self.toa)): @@ -651,8 +644,6 @@ def _fill_nan(self,): def fill_nan(array): x_shp, y_shp = array.shape mask = ~np.isnan(array) - if np.sum(mask) == 0: # no valid data - return np.full((x_shp, y_shp), np.nan) valid = np.array(np.where(mask)).T value = array[mask] mesh = np.repeat(range(x_shp), y_shp).reshape(x_shp, y_shp), \ @@ -679,73 +670,60 @@ def _solving(self,): self._parse_angles() self.logger.info('Mask bad pixels.') self._mask_bad_pix() - if np.sum(~self.bad_pix) > 10: - self.logger.info('Get simulated BOA.') - self._get_boa() - self.logger.info('Get PSF.') - self._get_psf() - self.logger.info('Get simulated TOA reflectance.') - self._get_convolved_toa() - self.logger.info('Filtering data.') - self._re_mask() - if self.mask is not False: - self.logger.info('Loading emulators.') - self._load_xa_xb_xc_emus() - self.logger.info('Reading priors and elevation.') - self._read_aux() - self._fill_nan() - self.logger.info('Mean values for prior AOT: %.02f and TCWV: %.02f'%(self._aot.mean(), self._tcwv.mean())) - if self.mask.sum() ==0: - self.logger.info('No valid value is found for retrieval of atmospheric parameters and priors are stored.') - ret = np.array([[self._aot, self._tcwv, self._tco3], [self._aot_unc, self._tcwv_unc, self._tco3_unc]]) - self.aero_res /=2 - #self.ySize *=2 - #self.xSize *=2 - self.ySize, self.xSize = self._aot.shape - else: - self.aero = solving_atmo_paras(self.boa, - self.toa, - self._sza, - self._vza, - self._saa, - self._vaa, - self._aot, - self._tcwv, - self._tco3, - self._ele, - self._aot_unc, - self._tcwv_unc, - self._tco3_unc, - self.boa_unc, - self.hx, self.hy, - self.mask, - self.full_res, - self.aero_res, - self.emus, - self.band_index, - self.boa_wv, - pix_res = self.pixel_res, - gamma = self.gamma, - log_file = self.log_file - ) - ret = self.aero._multi_grid_solver() - else: + self.logger.info('Get simulated BOA.') + self._get_boa() + self.logger.info('Get PSF.') + self._get_psf() + self.logger.info('Get simulated TOA reflectance.') + self._get_convolved_toa() + self.logger.info('Filtering data.') + self._re_mask() + if self.mask is not False: + self.logger.info('Loading emulators.') + self._load_xa_xb_xc_emus() + self.logger.info('Reading priors and elevation.') + self._read_aux() + self._fill_nan() + if self.mask.sum() ==0: self.logger.info('No valid value is found for retrieval of atmospheric parameters and priors are stored.') - self._read_aux() - self._fill_nan() - self.logger.info('Mean values for prior AOT: %.02f and TCWV: %.02f'%(self._aot.mean(), self._tcwv.mean())) ret = np.array([[self._aot, self._tcwv, self._tco3], [self._aot_unc, self._tcwv_unc, self._tco3_unc]]) self.aero_res /=2 #self.ySize *=2 #self.xSize *=2 self.ySize, self.xSize = self._aot.shape + else: + self.aero = solving_atmo_paras(self.boa, + self.toa, + self._sza, + self._vza, + self._saa, + self._vaa, + self._aot, + self._tcwv, + self._tco3, + self._ele, + self._aot_unc, + self._tcwv_unc, + self._tco3_unc, + self.boa_unc, + self.hx, self.hy, + self.mask, + self.full_res, + self.aero_res, + self.emus, + self.band_index, + self.boa_wv, + pix_res = self.pixel_res, + gamma = self.gamma) + ret = self.aero._multi_grid_solver() else: self.logger.info('No valid value is found for retrieval of atmospheric parameters and priors are stored.') self._read_aux() self._fill_nan() - self.logger.info('Mean values for prior AOT: %.02f and TCWV: %.02f'%(self._aot.mean(), self._tcwv.mean())) ret = np.array([[self._aot, self._tcwv, self._tco3], [self._aot_unc, self._tcwv_unc, self._tco3_unc]]) self.aero_res /=2 + #self.ySize *=2 + #self.xSize *=2 self.ySize, self.xSize = self._aot.shape solved = ret[0].reshape(3, self.ySize, self.xSize) @@ -758,10 +736,6 @@ def _solving(self,): parmap(par, name_arrays) self.post_aot, self.post_tcwv, self.post_tco3, = solved self.post_aot_unc, self.post_tcwv_unc, self.post_tco3_unc = unc - handlers = self.logger.handlers[:] - for handler in handlers: - handler.close() - self.logger.removeHandler(handler) def get_kk(angles): vza ,sza,raa = angles @@ -810,7 +784,7 @@ def test_s2(): def test_l8(): sensor_sat = 'OLI', 'L8' - base = '~/DATA/S2_MODIS/l_data/LC08_L1TP_014034_20170831_20170915_01_T1/LC08_L1TP_014034_20170831_20170915_01_T1_' + base = '/home/ucfafyi/DATA/S2_MODIS/l_data/LC08_L1TP_014034_20170831_20170915_01_T1/LC08_L1TP_014034_20170831_20170915_01_T1_' toa_bands = [base + i + '.TIF' for i in ['B2', 'B3', 'B4', 'B5', 'B6', 'B7']] view_angles = [base + 'sensor_%s'%i +'.tif' for i in ['B02', 'B03', 'B04', 'B05', 'B06', 'B07']] sun_angles = base + 'solar_B01.tif' @@ -877,3 +851,6 @@ def test_modis(): #l8_aero = test_l8() s2_aero = test_s2() #m_aero = test_modis() + + + diff --git a/SIAC/the_correction.py b/SIAC/the_correction.py index 0b579cc..830d8ea 100644 --- a/SIAC/the_correction.py +++ b/SIAC/the_correction.py @@ -1,6 +1,8 @@ #!/usr/bin/env python import os -import sys +import sys +import ogr +import gdal import psutil import logging import warnings @@ -12,13 +14,12 @@ import cPickle as pkl except: import pickle as pkl -from osgeo import gdal, ogr +#from osgeo import gdal from numpy import clip, uint8 from SIAC.multi_process import parmap from scipy.interpolate import griddata from scipy.ndimage import binary_dilation from SIAC.create_logger import create_logger -from SIAC.raster_boundary import get_boundary from SIAC.reproject import reproject_data, array_to_raster warnings.filterwarnings("ignore") @@ -43,7 +44,6 @@ def __init__(self, tco3_unc = None, cloud_mask = None, obs_time = None, - log_file = None, a_z_order = 1, ref_scale = 0.0001, ref_off = 0, @@ -52,6 +52,8 @@ def __init__(self, atmo_scale = [1., 1., 1., 1., 1., 1.], global_dem = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt', cams_dir = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/', + #global_dem = '/home/ucfafyi/DATA/Multiply/eles/global_dem.vrt', + #cams_dir = '/home/ucfafyi/netapp_10/cams/', emus_dir = 'SIAC/emus/', cams_scale = [1., 0.1, 46.698, 1., 1., 1.], block_size = 600, @@ -94,23 +96,30 @@ def __init__(self, if (r is not None) & (g is not None) & (b is not None): self.ri, self.gi, self.bi = self.toa_bands.index(r), self.toa_bands.index(g), self.toa_bands.index(b) self._do_rgb = True - - self.logger = create_logger(log_file) + ''' + # create logger + self.logger = logging.getLogger('SIAC') + self.logger.setLevel(logging.INFO) + if not self.logger.handlers: + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + ch.setFormatter(formatter) + self.logger.addHandler(ch) + ''' + self.logger = create_logger() def _create_base_map(self,): ''' Deal with different types way to define the AOI, if none is specified, then the image bound is used. ''' - gdal.UseExceptions() ogr.UseExceptions() + gdal.UseExceptions() if self.aoi is not None: if os.path.exists(self.aoi): try: g = gdal.Open(self.aoi) - #subprocess.call(['gdaltindex', '-f', 'GeoJSON', '-t_srs', 'EPSG:4326', self.toa_dir + '/AOI.json', self.aoi]) - geojson = get_boundary(self.aoi)[0] - with open(self.toa_dir + '/AOI.json', 'wb') as f: - f.write(geojson.encode()) + subprocess.call(['gdaltindex', '-f', 'GeoJSON', self.toa_dir + '/AOI.json', self.aoi]) except: try: gr = ogr.Open(self.aoi) @@ -139,22 +148,7 @@ def _create_base_map(self,): ogr.DontUseExceptions() gdal.DontUseExceptions() if not os.path.exists(self.toa_dir + '/AOI.json'): - #g = gdal.Open(self.toa_bands[0]) - #proj = g.GetProjection() - #if 'WGS 84' in proj: - # subprocess.call(['gdaltindex', '-f', 'GeoJSON', self.toa_dir +'/AOI.json', self.toa_bands[0]]) - #else: - # subprocess.call(['gdaltindex', '-f', 'GeoJSON', '-t_srs', 'EPSG:4326', self.toa_dir +'/AOI.json', self.toa_bands[0]]) - if 'WGS 84' in proj: - #subprocess.call(['gdaltindex', '-f', 'GeoJSON', self.toa_dir +'/AOI.json', self.toa_bands[0]]) - geojson = get_boundary(self.toa_bands[0], to_wgs84 = False) - with open(self.toa_dir + '/AOI.json', 'wb') as f: - f.write(geojson.encode()) - else: - #subprocess.call(['gdaltindex', '-f', 'GeoJSON', '-t_srs', 'EPSG:4326', self.toa_dir +'/AOI.json', self.toa_bands[0]]) - geojson = get_boundary(self.toa_bands[0])[0] - with open(self.toa_dir + '/AOI.json', 'wb') as f: - f.write(geojson.encode()) + subprocess.call(['gdaltindex', '-f', 'GeoJSON', self.toa_dir +'/AOI.json', self.toa_bands[0]]) self.logger.warning('AOI is not created and full band extend is used') self.aoi = self.toa_dir + '/AOI.json' else: @@ -229,20 +223,11 @@ def _annoying_angles(self,): _sun_angles = [] _view_angles = [] for fname in self._sun_angles: - #nodatas = [float(i.split("=")[1]) for i in gdal.Info(fname).split('\n') if' NoData' in i] - try: - nodatas = ' '.join([i.split("=")[1] for i in gdal.Info(fname).split('\n') if' NoData' in i]) - except: - nodatas = None - ang = reproject_data(fname, self.mg, srcNodata = nodatas, resample = \ + ang = reproject_data(fname, self.mg, srcNodata = None, resample = \ 0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data _sun_angles.append(ang) for fname in self._view_angles: - try: - nodatas = ' '.join([i.split("=")[1] for i in gdal.Info(fname).split('\n') if' NoData' in i]) - except: - nodatas = None - ang = reproject_data(fname, self.mg, srcNodata = nodatas, resample = \ + ang = reproject_data(fname, self.mg, srcNodata = None, resample = \ 0, dstNodata=np.nan, outputType= gdal.GDT_Float32).data _view_angles.append(ang) _view_angles = np.array(_view_angles) @@ -326,10 +311,8 @@ def _parse_aux(self,): def _fill_nan(self,): def fill_nan(array): - x_shp, y_shp = array.shape - mask = ~np.isnan(array) - if np.sum(mask) == 0: # all data is invalid - return np.full((x_shp, y_shp), np.nan) + x_shp, y_shp = array.shape + mask = ~np.isnan(array) valid = np.array(np.where(mask)).T value = array[mask] mesh = np.repeat(range(x_shp), y_shp).reshape(x_shp, y_shp), \ @@ -570,10 +553,6 @@ def _doing_correction(self,): self.logger.info('Composing RGB.') self._compose_rgb() self.logger.info('Done.') - handlers = self.logger.handlers[:] - for handler in handlers: - handler.close() - self.logger.removeHandler(handler) return ret def test_s2(): @@ -606,7 +585,7 @@ def test_s2(): return ret, atmo def test_l8(): sensor_sat = 'OLI', 'L8' - base = '~/DATA/S2_MODIS/l_data/LC08_L1TP_014034_20170831_20170915_01_T1/LC08_L1TP_014034_20170831_20170915_01_T1_' + base = '/home/ucfafyi/DATA/S2_MODIS/l_data/LC08_L1TP_014034_20170831_20170915_01_T1/LC08_L1TP_014034_20170831_20170915_01_T1_' toa_bands = [base + i + '.TIF' for i in ['B2', 'B3', 'B4', 'B5', 'B6', 'B7']] view_angles = [base + 'sensor_%s'%i +'.tif' for i in ['B02', 'B03', 'B04', 'B05', 'B06', 'B07']] sun_angles = base + 'solar_B01.tif' @@ -677,7 +656,7 @@ def test_modis(): aot_unc = toa_dir + '/aot_unc.tif' tcwv_unc = toa_dir + '/tcwv_unc.tif' tco3_unc = toa_dir + '/tco3_unc.tif' - emus_dir = '~/DATA/Multiply/emus/old_emus/' + emus_dir = '/data/store01/data_dirs/students/ucfafyi/Multiply/emus/old_emus/' rgb = [toa_bands[2], toa_bands[1], toa_bands[0]] atmo = atmospheric_correction(sensor_sat, toa_bands, band_index,view_angles,sun_angles, aot = aot, cloud_mask = cloud_mask, ref_scale = scale, ref_off = off,\ tcwv = tcwv, tco3 = tco3, aot_unc = aot_unc, tcwv_unc = tcwv_unc, tco3_unc = tco3_unc, rgb = rgb, emus_dir =emus_dir,block_size=3000) @@ -689,3 +668,9 @@ def test_modis(): #l_ret, l_atmo = test_l8() #m_ret, m_atmo = test_modis() + + + + + + From 6b0b031067bcc0280145ef728d94da31ae0a84c2 Mon Sep 17 00:00:00 2001 From: TonioF Date: Wed, 10 Apr 2019 11:51:57 +0200 Subject: [PATCH 102/103] revert to old state --- .readthedocs.yml | 7 - .travis.yml | 29 +- AUTHORS.rst | 1 - CHANGES.md | 0 MANIFEST.in | 3 - README.md | 28 +- SIAC/SIAC_S2.py | 39 +- SIAC/VERSION | 1 - SIAC/__init__.py | 1 - SIAC/atmo_solver.py | 64 +- SIAC/create_logger.py | 16 +- SIAC/downloaders.py | 10 +- SIAC/get_MCD43.py | 142 +-- SIAC/modis_tile_cal.py | 4 +- SIAC/psf_optimize.py | 16 +- SIAC/raster_boundary.py | 74 -- SIAC/reproject.py | 7 +- SIAC/s2_angle.py | 143 +-- SIAC/s2_preprocessing.py | 31 +- SIAC/tests/MCD43.txt | 33 - SIAC/tests/test_s2.py | 36 +- docs/Makefile | 19 - docs/make.bat | 35 - docs/source/01_Introduction.rst | 26 - docs/source/02_Installation.rst | 27 - docs/source/03_Quickstart.rst | 16 - ..._Data_access_and_adaption_for_Landsat5.rst | 1077 ----------------- docs/source/05_Annex_technical.rst | 48 - docs/source/authors.rst | 2 - docs/source/changes.rst | 2 - docs/source/conf.py | 203 ---- docs/source/index.rst | 55 - docs/source/requirements.txt | 1 - environment.yml | 2 +- requirements.txt | 2 +- setup.py | 14 +- 36 files changed, 175 insertions(+), 2039 deletions(-) delete mode 100644 .readthedocs.yml delete mode 100644 AUTHORS.rst delete mode 100644 CHANGES.md delete mode 100644 SIAC/VERSION delete mode 100644 SIAC/raster_boundary.py delete mode 100644 SIAC/tests/MCD43.txt delete mode 100644 docs/Makefile delete mode 100644 docs/make.bat delete mode 100644 docs/source/01_Introduction.rst delete mode 100644 docs/source/02_Installation.rst delete mode 100644 docs/source/03_Quickstart.rst delete mode 100644 docs/source/04_Data_access_and_adaption_for_Landsat5.rst delete mode 100644 docs/source/05_Annex_technical.rst delete mode 100644 docs/source/authors.rst delete mode 100644 docs/source/changes.rst delete mode 100644 docs/source/conf.py delete mode 100644 docs/source/index.rst delete mode 100644 docs/source/requirements.txt diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index 8be2dca..0000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,7 +0,0 @@ -build: - image: latest -conda: - file: environment.yml -python: - version: 3.6 - setup_py_install: true diff --git a/.travis.yml b/.travis.yml index b7a346e..16c922d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,11 +2,6 @@ language: python python: - "2.7" - "3.6" -matrix: - include: - - python: 3.7 - dist: xenial - sudo: true install: - sudo apt-get update - echo $TRAVIS_PYTHON_VERSION @@ -17,35 +12,33 @@ install: fi - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" + - hash -r - conda config --set always_yes yes --set changeps1 no - conda config --add channels conda-forge - conda update -q conda - - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six requests scikit-learn scikit-image 'gdal>=2.1,<2.4' pytest-cov coverage pip + - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytest pyyaml scipy numpy psutil six lightgbm requests scikit-learn scikit-image gdal=2.2.4 coveralls pytest-cov coverage=4.5.1 h5py - source activate test-environment - conda info -a + - conda list + - conda install pip - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then - pip install https://github.com/MarcYin/gp_emulator/archive/master.zip; + pip install https://github.com/MarcYin/gp_emulator/archive/62f288792e1b24a12891fc0ad32896cb7cb5eda4.zip; else pip install gp_emulator==1.6.1; fi - - pip install lightgbm >=2.1.0 coveralls codecov - - conda list + - pip install coveralls + - pip install codecov script: # Output something every 10 minutes or Travis kills the job - while sleep 9m; do echo "=====[ $SECONDS seconds still running ]====="; done & - py.test -s --cov #Killing background sleep loop - kill %1 - - cp /dev/null SIAC/data/.earthdata_auth + #- travis_wait 60 py.test -s --cov - coverage xml - #- export SIAC_VERSION=$TRAVIS_TAG -deploy: - provider: pypi - user: $PYPI_USER - password: $PYPI_PASS - skip_existing: true - on: - branch: master after_success: - codecov - coveralls + - rm -rf SIAC/emus/*.pkl + - rm -rf S2?_MSIL1C*.SAFE + - test $TRAVIS_BRANCH = "master" && conda install conda-build && conda install anaconda-client && PKG_NAME=siac && USER=f0xy && OS=$TRAVIS_OS_NAME-64 && mkdir ~/conda-bld && conda config --set anaconda_upload no && export CONDA_BLD_PATH=~/conda-bld && export VERSION=`date +%Y.%m.%d` && conda build . -c conda-forge && ls -lah $CONDA_BLD_PATH/$OS && anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l nightly $(ls $CONDA_BLD_PATH/$OS/$PKG_NAME-$VERSION*.tar.bz2) --force diff --git a/AUTHORS.rst b/AUTHORS.rst deleted file mode 100644 index c47465e..0000000 --- a/AUTHORS.rst +++ /dev/null @@ -1 +0,0 @@ -* Feng Yin<"ucfafyi@ucl.ac.uk"> diff --git a/CHANGES.md b/CHANGES.md deleted file mode 100644 index e69de29..0000000 diff --git a/MANIFEST.in b/MANIFEST.in index 8e2a3c8..96091c1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,6 @@ include setup.py -include SIAC/VERSION include SIAC/util/* include SIAC/data/sen2cloud_detector.pkl include SIAC/data/.earthdata_auth include SIAC/spectral_mapping/* include SIAC/tests/*.py -include SIAC/tests/aoi.geojson -include SIAC/tests/s2_flists.txt diff --git a/README.md b/README.md index 533b56f..4ae7543 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,9 @@ [![conda](https://anaconda.org/f0xy/siac/badges/version.svg?longCache=true&style=flat)](https://anaconda.org/F0XY/siac) [![py version](https://img.shields.io/pypi/pyversions/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) [![build](https://img.shields.io/travis/MarcYin/SIAC/master.svg?longCache=true&style=flat)](https://travis-ci.org/MarcYin/SIAC/) -[![Documentation Status](https://readthedocs.org/projects/siac/badge/?version=latest)](https://siac.readthedocs.io/en/latest/?badge=latest) [![codecov](https://codecov.io/gh/MarcYin/SIAC/branch/master/graph/badge.svg?longCache=true&style=flat)](https://codecov.io/gh/MarcYin/SIAC) [![Coverage Status](https://coveralls.io/repos/github/MarcYin/SIAC/badge.svg?branch=master)](https://coveralls.io/github/MarcYin/SIAC?branch=master) [![Lisence](https://img.shields.io/pypi/l/siac.svg?longCache=true&style=flat)](https://pypi.org/project/SIAC/) -[![DOI](https://zenodo.org/badge/117815245.svg)](https://zenodo.org/badge/latestdoi/117815245) This atmospheric correction method uses MODIS MCD43 BRDF product to get a coarse resolution simulation of earth surface. A model based on MODIS PSF is built to deal with the scale differences between MODIS and Sentinel 2 / Landsat 8. We uses the ECMWF CAMS prediction as a prior for the atmospheric states, coupling with 6S model to solve for the atmospheric parameters. We do not have topography correction and homogeneouse surface is used without considering the BRDF effects. @@ -25,31 +23,13 @@ This atmospheric correction method uses MODIS MCD43 BRDF product to get a coarse ## Installation: -1. Directly from github +Download this repositories either by `git clone git@ github.com:MarcYin/SIAC.git` or download the zip file and unzip it. In the main directory of it excute `pip install .` , or `pip install SIAC` or `conda install -c f0xy siac` -```bash -pip install https://github.com/multiply-org/atmospheric_correction/archive/master.zip -``` - - -2. Using PyPI - -```bash -pip install SIAC -``` - - -3. Using anaconda - -```bash -conda install -c f0xy -c conda-forge siac -``` - - -To save your time for installing GDAL: +To save your time for installing GDAL and lightgbm, please use: ```bash -conda install -c conda-forge gdal>2.1 +conda install -c conda-forge gdal=2.2.4 +conda install -c conda-forge lightgbm ``` diff --git a/SIAC/SIAC_S2.py b/SIAC/SIAC_S2.py index 0a78ad4..a474006 100755 --- a/SIAC/SIAC_S2.py +++ b/SIAC/SIAC_S2.py @@ -1,25 +1,31 @@ -import os -import sys import argparse +import os import requests -import warnings -warnings.filterwarnings("ignore") import numpy as np from glob import glob from SIAC.get_MCD43 import get_mcd43, get_local_MCD43 from datetime import datetime from SIAC.the_aerosol import solve_aerosol -from SIAC.create_logger import create_logger from SIAC.the_correction import atmospheric_correction from SIAC.s2_preprocessing import s2_pre_processing from SIAC.downloaders import downloader from SIAC.multi_process import parmap +from SIAC.create_logger import create_logger from os.path import expanduser home = expanduser("~") file_path = os.path.dirname(os.path.realpath(__file__)) logger = create_logger() +command_2 = ['python', '/home/tonio-bc/projects/multiply/atmospheric_correction/SIAC/SIAC_S2.py', + '-f', '/home/tonio-bc/EOData/m1/s2/2017-06-01/30/S/WJ/2017/6/5/0/', '-m', '/home/tonio-bc/EOData/m1/modis/2017-06-01' + '-e', '/home/tonio-bc/EOData/m1/emulators', '-c', '/home/tonio-bc/EOData/m1/cams/2017-06-01', + '-d', '/home/tonio-bc/EOData/dems/aster_dem.vrt', '-o', 'False', + '-a', "'POLYGON((-2.20397502663252 39.09868106889479,-2.1142106223355313 39.09868106889479," + "-2.1142106223355313 38.94504502508093,-2.20397502663252 38.94504502508093," + "-2.20397502663252 39.09868106889479))'"] + + def _ensure_dir_format(dir: str): if not dir.endswith('/'): @@ -79,22 +85,10 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask obs_time = datetime.strptime(sensing_time, u'%Y-%m-%dT%H:%M:%S.%fZ') if 'TILE_ID' in i: sat = i.split('')[-1].split('_')[0] - tile = i.split('')[-1] - log_file = os.path.dirname(metafile) + '/SIAC_S2.log' - logger = create_logger(log_file) - logger.info('Starting atmospheric corretion for %s' % tile) - if not np.all(cloud_mask): - handlers = logger.handlers[:] - for handler in handlers: - handler.close() - logger.removeHandler(handler) - if download_mcd_43: - get_mcd43(toa_refs[0], obs_time, mcd43_dir=mcd43, vrt_dir=vrt_dir, log_file=log_file) - else: - get_local_MCD43(toa_refs[0], obs_time, mcd43_dir=mcd43, vrt_dir=vrt_dir) - # logger = create_logger(log_file) + if download_mcd_43: + get_mcd43(toa_refs[0], obs_time, mcd43_dir = mcd43, vrt_dir = vrt_dir) else: - logger.info('No clean pixel in this scene and no MCD43 is downloaded.') + get_local_MCD43(toa_refs[0], obs_time, mcd43_dir=mcd43, vrt_dir=vrt_dir) sensor_sat = 'MSI', sat band_index = [1,2,3,7,11,12] band_wv = [469, 555, 645, 859, 1640, 2130] @@ -104,7 +98,7 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,view_angles,\ sun_angles,obs_time,cloud_mask, gamma=10., spec_m_dir= \ file_path+'/spectral_mapping/', emus_dir=emus_dir, mcd43_dir=vrt_dir, aoi=aoi, - global_dem=dem_vrt, cams_dir=cams_dir, log_file = log_file) + global_dem=dem_vrt, cams_dir=cams_dir) aero._solving() toa_bands = toa_refs view_angles = view_ang_names @@ -120,10 +114,9 @@ def do_correction(sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask sun_angles, aot = aot, cloud_mask = cloud_mask,\ tcwv = tcwv, tco3 = tco3, aot_unc = aot_unc, \ tcwv_unc = tcwv_unc, tco3_unc = tco3_unc, rgb = \ - rgb, emus_dir=emus_dir, global_dem=dem_vrt, cams_dir=cams_dir, log_file = log_file) + rgb, emus_dir=emus_dir, global_dem=dem_vrt, cams_dir=cams_dir) atmo._doing_correction() return aero, atmo - if __name__ == '__main__': parser = argparse.ArgumentParser(description='Sentinel 2 Atmospheric Correction Excutable') parser.add_argument('-f', "--file_path", help='Sentinel 2 file path in the form of AWS', required=True) diff --git a/SIAC/VERSION b/SIAC/VERSION deleted file mode 100644 index ccbccc3..0000000 --- a/SIAC/VERSION +++ /dev/null @@ -1 +0,0 @@ -2.2.0 diff --git a/SIAC/__init__.py b/SIAC/__init__.py index 3a5f343..236763d 100644 --- a/SIAC/__init__.py +++ b/SIAC/__init__.py @@ -1,6 +1,5 @@ from .the_aerosol import solve_aerosol from .the_correction import atmospheric_correction -from .atmo_solver import solving_atmo_paras from .SIAC_S2 import SIAC_S2 from .SIAC_L8 import SIAC_L8 diff --git a/SIAC/atmo_solver.py b/SIAC/atmo_solver.py index 98356c4..a5407f8 100644 --- a/SIAC/atmo_solver.py +++ b/SIAC/atmo_solver.py @@ -12,8 +12,14 @@ import pickle as pkl from scipy import optimize, interpolate, sparse from scipy.sparse import linalg +#from fastDiff import fastDiff from SIAC.multi_process import parmap +#turn off multithreading +#os.environ['OPENBLAS_NUM_THREADS'] = '1' +#os.environ['MKL_NUM_THREADS'] = '1' +#os.environ['OMP_NUM_THREADS'] = '1' + class bcolors: HEADER = '\033[95m' BLUE = '\033[94m' @@ -68,8 +74,7 @@ def __init__(self, gamma = 0.5, alpha = -1.6,# from nasa modis climatology subsample = 1, - subsample_start = 0, - log_file = None + subsample_start = 0 ): self.boa = boa @@ -111,11 +116,6 @@ def __init__(self, formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) self.logger.addHandler(ch) - if log_file is not None: - fh = logging.FileHandler(log_file) - fh.setLevel(logging.DEBUG) - fh.setFormatter(formatter) - self.logger.addHandler(fh) self.logger.propagate = False self.logger.info('MultiGrid solver in process...') @@ -257,10 +257,6 @@ def _multi_grid_solver(self,): self.tco3_prior, self.tco3_unc = self._grid_conversion(self.tco3_prior, shape), self._grid_conversion(self.tco3_unc, shape) post_solved = np.array([self.aot_prior, self.tcwv_prior, self.tco3_prior]) post_unc = np.array([self.aot_unc, self.tcwv_unc, self.tco3_unc]) - handlers = self.logger.handlers[:] - for handler in handlers: - handler.close() - self.logger.removeHandler(handler) return [post_solved, post_unc] def _helper(self, inp): @@ -304,40 +300,20 @@ def _obs_cost_test(self, p, is_full = True, do_unc=False): if is_full: dJ = np.nansum(np.array(full_dJ), axis=(1,)) - dJ [np.isnan(dJ)] = 0 - full_J[np.isnan(full_J)] = 0 - #J_ = np.zeros((2,) + self.full_res) - #J_[:, self.Hx, self.Hy] = dJ - #subs1 = [np.array_split(sub, self.num_blocks_y, axis=2) for sub in np.array_split(J_, self.num_blocks_x, axis=1)] - - #J = np.zeros(self.full_res) - #J[self.Hx, self.Hy] = full_J - #subs2 = [np.array_split(sub, self.num_blocks_y, axis=1) for sub in np.array_split(J, self.num_blocks_x, axis=0)] - - #J_ = np.zeros((2, self.num_blocks_x, self.num_blocks_y)) - #J = np.zeros(( self.num_blocks_x, self.num_blocks_y)) - - nx, ny = (np.ceil(np.array(self.full_res) / np.array([self.num_blocks_x, self.num_blocks_y])) \ - * np.array([self.num_blocks_x, self.num_blocks_y])).astype(int) - #end_x, end_y = np.array(self.full_res) - np.array([nx, ny]) - x_size, y_size = int(nx / self.num_blocks_x), int(ny / self.num_blocks_y) - - J_ = np.zeros((2, nx, ny)) - J = np.zeros(( nx, ny)) - #J_[:], J[:] = np.nan, np.nan + J_ = np.zeros((2,) + self.full_res) + J_[:, self.Hx, self.Hy] = dJ + subs1 = [np.array_split(sub, self.num_blocks_y, axis=2) for sub in np.array_split(J_, self.num_blocks_x, axis=1)] - J_[:, self.Hx, self.Hy] = dJ - J [ self.Hx, self.Hy] = full_J + J = np.zeros(self.full_res) + J[self.Hx, self.Hy] = full_J + subs2 = [np.array_split(sub, self.num_blocks_y, axis=1) for sub in np.array_split(J, self.num_blocks_x, axis=0)] - J_ = J_.reshape(2, self.num_blocks_x, x_size, self.num_blocks_y, y_size) - J = J.reshape( self.num_blocks_x, x_size, self.num_blocks_y, y_size) - J_ = np.sum(J_, axis=(2,4)) - J = np.sum(J, axis=(1,3)) - - #for i in range(self.num_blocks_x): - # for j in range(self.num_blocks_y): - # J_[:, i,j] = np.nansum(subs1[i][j], axis=(1,2)) - # J [ i,j] = np.nansum(subs2[i][j], axis=(0,1)) + J_ = np.zeros((2, self.num_blocks_x, self.num_blocks_y)) + J = np.zeros(( self.num_blocks_x, self.num_blocks_y)) + for i in range(self.num_blocks_x): + for j in range(self.num_blocks_y): + J_[:, i,j] = np.nansum(subs1[i][j], axis=(1,2)) + J [ i,j] = np.nansum(subs2[i][j], axis=(0,1)) J_[:, ~self._coarse_mask] = 0 J [ ~self._coarse_mask] = 0 @@ -469,7 +445,7 @@ def _cost(self, p): Hy = np.random.choice(10980, 50000) full_res = (10980, 10980) aero_res = 600 - emus_dir = '~/DATA/Multiply/emus/' + emus_dir = '/home/ucfafyi/DATA/Multiply/emus/' sensor = 'MSI' xap_emu = glob(emus_dir + '/isotropic_%s_emulators_*xap*.pkl'%(sensor))[0] xbp_emu = glob(emus_dir + '/isotropic_%s_emulators_*xbp*.pkl'%(sensor))[0] diff --git a/SIAC/create_logger.py b/SIAC/create_logger.py index c16c142..fc503cd 100644 --- a/SIAC/create_logger.py +++ b/SIAC/create_logger.py @@ -1,21 +1,11 @@ -import os import logging -file_path = os.path.dirname(os.path.realpath(__file__)) -version_file = file_path + '/VERSION' -with open(version_file, 'rb') as f: - version = f.read().decode().strip() -def create_logger(fname = None): - logger = logging.getLogger('SIAC-V%s'%version) +def create_logger(): + logger = logging.getLogger('SIAC') logger.setLevel(logging.INFO) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') if not logger.handlers: ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) logger.addHandler(ch) - if fname is not None: - fh = logging.FileHandler(fname) - fh.setLevel(logging.DEBUG) - fh.setFormatter(formatter) - logger.addHandler(fh) return logger diff --git a/SIAC/downloaders.py b/SIAC/downloaders.py index 1c7b280..5fc3d78 100644 --- a/SIAC/downloaders.py +++ b/SIAC/downloaders.py @@ -17,12 +17,12 @@ def downloader(fname, url_root, file_dir): logger.propagate = False new_url = url_root + fname - new_req = requests.get(new_url) + new_req = requests.get(new_url, stream=True) if new_req.ok: logger.info('downloading %s and save it at %s' % (fname, os.path.join(file_dir, fname))) with open(os.path.join(file_dir, fname), 'wb') as fp: - #for chunk in new_req.iter_content(chunk_size=1024): - # if chunk: - fp.write(new_req.content) + for chunk in new_req.iter_content(chunk_size=1024): + if chunk: + fp.write(chunk) else: - logger.error(new_req.content) + logger.error('Requests failed.') diff --git a/SIAC/get_MCD43.py b/SIAC/get_MCD43.py index f0cf6d6..e90fe04 100644 --- a/SIAC/get_MCD43.py +++ b/SIAC/get_MCD43.py @@ -2,108 +2,24 @@ import time import gdal import getpass +import logging import requests import numpy as np from glob import glob -from six.moves import input from functools import partial from multiprocessing import Pool from datetime import datetime, timedelta +from SIAC.modis_tile_cal import get_vector_hv, get_raster_hv from os.path import expanduser from SIAC.create_logger import create_logger -from SIAC.modis_tile_cal import get_vector_hv, get_raster_hv +from six.moves import input home = expanduser("~") file_path = os.path.dirname(os.path.realpath(__file__)) logger = create_logger() -test_url = 'https://e4ftl01.cr.usgs.gov/MOTA/MCD43A1.006/2000.02.24/MCD43A1.A2000055.h32v08.006.2016101152216.hdf.xml' - -def get_auth(logger): - try: - auth = tuple([os.environ['Earthdata_user'], os.environ['Earthdata_pass']]) - with requests.Session() as s: - s.auth = auth - r1 = s.get(test_url) - r = s.get(r1.url, stream=True, headers={'user-agent': 'My app'}) - if r.status_code == 401: - logger.error('Wrong username and password are set for Earthdata_user and Earthdata_pass Environment variables.') - else: - with open(file_path + '/data/.earthdata_auth', 'wb') as f: - for i in auth: - f.write((i + '\n').encode()) - except: - #logger.error('Environment variables Earthdata_user and Earthdata_pass for accessing NASA Earthdata are not set, and trying to read from file or input.') - pass - if os.path.exists(file_path + '/data/.earthdata_auth'): - try: - username, password = np.loadtxt(file_path + '/data/.earthdata_auth', dtype=str) - auth = tuple([username, password]) - with requests.Session() as s: - s.auth = auth - r1 = s.get(test_url) - r = s.get(r1.url, stream=True, headers={'user-agent': 'My app'}) - while r.status_code == 401: - logger.error('Wrong username and password stored, please enter again') - username = input('Username for NASA Earthdata: ') - password = getpass.getpass('Password for NASA Earthdata: ') - auth = tuple([username, password]) - with requests.Session() as s: - s.auth = auth - r1 = s.get(test_url) - r = s.get(r1.url, stream=True, headers={'user-agent': 'My app'}) - - os.remove(file_path + '/data/.earthdata_auth') - with open(file_path + '/data/.earthdata_auth', 'wb') as f: - for i in auth: - f.write((i + '\n').encode()) - auth = tuple([username, password]) - - except: - logger.error('Please provide NASA Earthdata username and password for downloading MCD43 data, which can be applied here: https://urs.earthdata.nasa.gov.') - username = input('Username for NASA Earthdata: ') - password = getpass.getpass('Password for NASA Earthdata: ') - auth = username, password - with requests.Session() as s: - s.auth = auth - r1 = s.get(test_url) - r = s.get(r1.url, stream=True, headers={'user-agent': 'My app'}) - while r.status_code == 401: - logger.error('Wrong username and password typed, please enter again') - username = input('Username for NASA Earthdata: ') - password = getpass.getpass('Password for NASA Earthdata: ') - auth = tuple([username, password]) - with requests.Session() as s: - s.auth = auth - r1 = s.get(test_url) - r = s.get(r1.url, stream=True, headers={'user-agent': 'My app'}) - os.remove(file_path + '/data/.earthdata_auth') - with open(file_path + '/data/.earthdata_auth', 'wb') as f: - for i in auth: - f.write((i + '\n').encode()) - auth = tuple([username, password]) - else: - username = input('Username for NASA Earthdata: ') - password = getpass.getpass('Password for NASA Earthdata: ') - auth = username, password - with requests.Session() as s: - s.auth = auth - r1 = s.get(test_url) - r = s.get(r1.url, stream=True, headers={'user-agent': 'My app'}) - while r.status_code == 401: - logger.error('Wrong username and password typed, please enter again') - username = input('Username for NASA Earthdata: ') - password = getpass.getpass('Password for NASA Earthdata: ') - auth = tuple([username, password]) - with requests.Session() as s: - s.auth = auth - r1 = s.get(test_url) - r = s.get(r1.url, stream=True, headers={'user-agent': 'My app'}) - with open(file_path + '/data/.earthdata_auth', 'wb') as f: - for i in auth: - f.write((i + '\n').encode()) - auth = tuple([username, password]) - return auth +#print(file_path + '/data/.earthdata_auth') +#print(auth) def find_local_files(aoi, obs_time, temporal_window = 16): @@ -136,7 +52,7 @@ def find_files(aoi, obs_time, temporal_window = 16): except: raise IOError('AOI has to be raster or vector object/files.') fls = zip(np.repeat(tiles, len(days)), np.tile(days, len(tiles))) - p = Pool(8) + p = Pool(18) ret = p.map(get_one_tile, fls) p.close() p.join() @@ -156,9 +72,31 @@ def get_one_tile(tile_date): time.sleep(1) return base + date + '/' + fname[0] -def downloader(url_fname, auth): + +def downloader(url_fname): + if os.path.exists(file_path + '/data/.earthdata_auth'): + try: + username, password = np.loadtxt(file_path + '/data/.earthdata_auth', dtype=str) + auth = tuple([username, password]) + except: + logger.error('Please provide NASA Earthdata username and password for downloading MCD43 data, which can be applied here: https://urs.earthdata.nasa.gov.') + username = input('Username for NASA Earthdata: ') + password = getpass.getpass('Password for NASA Earthdata: ') + up = username, password + os.remove(file_path + '/data/.earthdata_auth') + with open(file_path + '/data/.earthdata_auth', 'wb') as f: + for i in up: + f.write((i + '\n').encode()) + auth = tuple([username, password]) + else: + username = input('Username for NASA Earthdata: ') + password = getpass.getpass('Password for NASA Earthdata: ') + up = username, password + with open(home + '/.earthdata_auth', 'wb') as f: + for i in up: + f.write((i + '\n').encode()) + auth = tuple([username, password]) url, fname = url_fname - fname = os.path.abspath(fname) with requests.Session() as s: s.max_redirects = 100000 s.auth = auth @@ -193,7 +131,6 @@ def daily_vrt(fnames_date, vrt_dir = None): fnames, date = fnames_date fnames = map(os.path.abspath, fnames) - #fnames = list(map(os.path.abspath, fnames)) all_files = [] for fname in fnames: @@ -235,10 +172,7 @@ def get_local_MCD43(aoi, obs_time, mcd43_dir ='./MCD43/', vrt_dir ='./MCD43_VRT/ daily_vrt(fname_date, vrt_dir) -def get_mcd43(aoi, obs_time, mcd43_dir = './MCD43/', vrt_dir = './MCD43_VRT/', log_file = None): - mcd43_dir = os.path.expanduser(mcd43_dir) # incase ~ used - vrt_dir = os.path.expanduser(vrt_dir) - logger = create_logger(log_file) +def get_mcd43(aoi, obs_time, mcd43_dir = './MCD43/', vrt_dir = './MCD43_VRT/'): logger.propagate = False logger.info('Start downloading MCD43 for the AOI, this may take some time.') logger.info('Query files...') @@ -246,9 +180,7 @@ def get_mcd43(aoi, obs_time, mcd43_dir = './MCD43/', vrt_dir = './MCD43_VRT/', l url_fnames = [[i, mcd43_dir + '/' + i.split('/')[-1]] for i in ret] p = Pool(5) logger.info('Start downloading...') - auth = get_auth(logger) - par = partial(downloader, auth = auth) - ret = p.map(par, url_fnames) + ret = p.map(downloader, url_fnames) p.close() p.join() flist = np.array(url_fnames)[:,1] @@ -261,14 +193,10 @@ def get_mcd43(aoi, obs_time, mcd43_dir = './MCD43/', vrt_dir = './MCD43_VRT/', l p.map(par, fnames_dates) p.close() p.join() - handlers = logger.handlers[:] - for handler in handlers: - handler.close() - logger.removeHandler(handler) if __name__ == '__main__': - aoi = '~/DATA/S2_MODIS/l_data/LC08_L1TP_014034_20170831_20170915_01_T1/AOI.json' - aoi = '~/DATA/S2_MODIS/l_data/LC08_L1TP_014034_20170831_20170915_01_T1/aot.tif' + aoi = '/home/ucfafyi/DATA/S2_MODIS/l_data/LC08_L1TP_014034_20170831_20170915_01_T1/AOI.json' + aoi = '/home/ucfafyi/DATA/S2_MODIS/l_data/LC08_L1TP_014034_20170831_20170915_01_T1/aot.tif' obs_time = datetime(2017, 7, 8, 10, 8, 20) - ret = get_mcd43(aoi, obs_time, mcd43_dir = '~/hep/MCD43/', vrt_dir = '~/DATA/Multiply/MCD43/') + ret = get_mcd43(aoi, obs_time, mcd43_dir = '/home/ucfafyi/hep/MCD43/', vrt_dir = '/home/ucfafyi/DATA/Multiply/MCD43/') diff --git a/SIAC/modis_tile_cal.py b/SIAC/modis_tile_cal.py index 065d3b4..1004442 100644 --- a/SIAC/modis_tile_cal.py +++ b/SIAC/modis_tile_cal.py @@ -87,8 +87,8 @@ def mtile_cal(lat, lon): return h,v if __name__ == '__main__': - aoi = '~/DATA/S2_MODIS/l_data/LC08_L1TP_014034_20170831_20170915_01_T1/AOI.json' - example_file = '~/DATA/S2_MODIS/l_data/LC08_L1TP_014034_20170831_20170915_01_T1/aot.tif' + aoi = '/home/ucfafyi/DATA/S2_MODIS/l_data/LC08_L1TP_014034_20170831_20170915_01_T1/AOI.json' + example_file = '/home/ucfafyi/DATA/S2_MODIS/l_data/LC08_L1TP_014034_20170831_20170915_01_T1/aot.tif' print(get_vector_hv(aoi)) print(get_vector_hv(ogr.Open(aoi))) print(get_raster_hv(example_file)) diff --git a/SIAC/psf_optimize.py b/SIAC/psf_optimize.py index ac1e284..eba5f43 100644 --- a/SIAC/psf_optimize.py +++ b/SIAC/psf_optimize.py @@ -51,26 +51,18 @@ def _preprocess(self,): self.bad_pixs = cloud_dilation( (self.high_img < 0.0001) | self.cloud | (self.high_img >= 1), iteration=int(size/2)) #xstd, ystd = 29.75, 39 #ker = self.gaussian(self.xstd, self.ystd, 0) + gaus_2d = self.dct_gaussian(self.xstd, self.ystd, self.high_img.shape) if (self.xstd < 1.) or (self.ystd < 1.): self.conved = self.high_img else: - data = self._pad_even_shape(self.high_img) - gaus_2d = self.dct_gaussian(self.xstd, self.ystd, data.shape) - self.conved = idct(idct(dct(dct(data, axis=0, norm = 'ortho'), axis=1, norm='ortho') * gaus_2d, \ - axis=1, norm = 'ortho'), axis=0, norm='ortho') + self.conved = idct(idct(dct(dct(self.high_img, axis=0, norm = 'ortho'), axis=1, norm='ortho') * gaus_2d, \ + axis=1, norm = 'ortho'), axis=0, norm='ortho') #self.conved = signal.fftconvolve(self.high_img, ker, mode='same') + l_mask = (~self.low_img.mask) & (self.qa')[0].replace('"', '').replace('>', '') - geojson = { - "type": "FeatureCollection", - "name": "AOI", - "crs": { "type": "name", "properties": { "name": urn} }, - "features": [ - { "type": "Feature", "properties": {}, "geometry": spolygon} - ] - } - except: - proj4 = sourceprj.ExportToProj4() - geojson = { - "type": "FeatureCollection", - "name": "AOI", - "crs": { "type": "name", "properties": { "name": proj4, "type": "proj4"} }, - "features": [ - { "type": "Feature", "properties": {}, "geometry": spolygon} - ] - } - - geojson = json.dumps(geojson) - - if to_wgs84: - targetprj = osr.SpatialReference() - targetprj.ImportFromEPSG(4326) - transform = osr.CoordinateTransformation(sourceprj, targetprj) - geometry.Transform(transform) - - stgeometry = geometry.SimplifyPreserveTopology(abs(geom[1]) / 10. * 0.000001) - stpolygon = json.loads(stgeometry.ExportToJson()) - - tgeojson = { - "type": "FeatureCollection", - "name": "AOI", - "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::4326"} }, - "features": [ - { "type": "Feature", "properties": {}, "geometry": stpolygon} - ] - } - tgeojson = json.dumps(tgeojson) - - return tgeojson, geojson - else: - return geojson - diff --git a/SIAC/reproject.py b/SIAC/reproject.py index b8663c5..f2067bc 100644 --- a/SIAC/reproject.py +++ b/SIAC/reproject.py @@ -45,11 +45,6 @@ def __init__(self, source_img, self.xSize = xSize self.ySize = ySize self.resample = resample - if self.srcNodata is None: - try: - self.srcNodata = ' '.join([i.split("=")[1] for i in gdal.Info(self.source_img).split('\n') if' NoData' in i]) - except: - self.srcNodata = None if (self.target_img is None) & (self.dstSRS is None): raise IOError('Projection should be specified ether from a file or a projection code.') elif self.target_img is not None: @@ -119,7 +114,7 @@ def array_to_raster(array, example_file): if __name__=='__main__': - ele = reproject_data('~/DATA/Multiply/eles/global_dem.vrt','/data/nemesis/S2_data/32/U/PU/2017/12/15/0/B02.jp2', outputType= gdal.GDT_Float32, ) + ele = reproject_data('/home/ucfafyi/DATA/Multiply/eles/global_dem.vrt','/data/nemesis/S2_data/32/U/PU/2017/12/15/0/B02.jp2', outputType= gdal.GDT_Float32, ) #ele.get_it() mask = (ele.data == -32768) | (~np.isfinite(ele.data)) ele.data = ma.array(ele.data, mask = mask) diff --git a/SIAC/s2_angle.py b/SIAC/s2_angle.py index 41df764..9a74fba 100644 --- a/SIAC/s2_angle.py +++ b/SIAC/s2_angle.py @@ -11,7 +11,6 @@ from functools import partial from multiprocessing import Pool import xml.etree.ElementTree as ET -from scipy.ndimage import binary_dilation #from scipy.interpolate import griddata from SIAC.create_logger import create_logger ''' @@ -121,93 +120,83 @@ def get_angle(view_ang_name_gml, vaa, vza, band_dict): va2 = None vz2 = None try: - dets = [] - for i in range(layer.GetFeatureCount()): - dets.append(layer.GetFeature(i).items()['gml_id']) - dets = sorted(dets, key = lambda i: int(i.split('-')[2])) - for i in range(len(dets)): - det = dets[i] - foot1 = gdal.Rasterize("", gml, format="MEM", xRes=xRes, yRes=yRes, \ - where="gml_id='%s'"%det, outputBounds=[ x_min, y_min, x_max, y_max], \ - noData=0, burnValues=1, outputType=gdal.GDT_Byte).ReadAsArray() - foot1 = binary_dilation(foot1) + for i in range(layer.GetFeatureCount()): + det = layer.GetFeature(i).items()['gml_id'] + foot1 = gdal.Rasterize("", gml, format="MEM", xRes=xRes, yRes=yRes, where="gml_id='%s'"%det, outputBounds=[ x_min, y_min, x_max, y_max], noData=0, burnValues=1, outputType=gdal.GDT_Byte).ReadAsArray() key = band_dict[det.split('-')[-3]], int(det.split('-')[-2]) va1 = vaa[key] vz1 = vza[key] if i>0: overlap = foot1 * foot2 - if overlap.sum() < 10: - foot1 = foot2 + x,y = np.where(overlap) + xmin, xmax, ymin, ymax = x.min(), x.max(), y.min(),y.max() + ll = x[x==xmax][-1], y[x==xmax][-1] + lr = x[y==ymax][-1], y[y==ymax][-1] + ul = x[y==ymin][0], y[y==ymin][0] + ur = x[x==xmin][0], y[x==xmin][0] + p1 = np.mean([lr, ur], axis=0) + p2 = np.mean([ll, ul], axis=0) + x1,y1 = np.where(foot2) + vamax, vamin = np.nanmax(va2), np.nanmin(va2) + vzmax, vzmin = np.nanmax(vz2), np.nanmin(vz2) + if not (p1==p2).all(): + p = np.poly1d(np.polyfit([p1[1], p2[1]],[p1[0], p2[0]],1)) + foot2[x[x > p(y)], y[x > p(y)]] = False + minx, miny = np.where(va2 == vamin) + maxx, maxy = np.where(va2 == vamax) + min_dst = abs(p(miny*y_scale)-minx*x_scale)/(np.sqrt(1+p.c[0]**2)) + max_dst = abs(p(maxy*y_scale)-maxx*x_scale)/(np.sqrt(1+p.c[0]**2)) + if (max_dst < min_dst).any(): + tmp1 = vamin.copy() + vamin = vamax + vamax = tmp1 + minx, miny = np.where(vz2 == vzmin) + maxx, maxy = np.where(vz2 == vzmax) + min_dst = abs(p(miny*y_scale)-minx*x_scale)/(np.sqrt(1+p.c[0]**2)) + max_dst = abs(p(maxy*y_scale)-maxx*x_scale)/(np.sqrt(1+p.c[0]**2)) + if (max_dst < min_dst).any(): + tmp2 = vzmin.copy() + vzmin = vzmax + vzmax = tmp2 + dist = abs(p(y1)-x1)/(np.sqrt(1+p.c[0]**2)) + vas[x1,y1] = vamin + dist/(dist.max()-dist.min()) * (vamax-vamin) + vzs[x1,y1] = vzmin + dist/(dist.max()-dist.min()) * (vzmax-vzmin) else: - x,y = np.where(overlap) - xmin, xmax, ymin, ymax = x.min(), x.max(), y.min(),y.max() - ll = x[x==xmax][-1], y[x==xmax][-1] - lr = x[y==ymax][-1], y[y==ymax][-1] - ul = x[y==ymin][0], y[y==ymin][0] - ur = x[x==xmin][0], y[x==xmin][0] - p1 = np.mean([lr, ur], axis=0) - p2 = np.mean([ll, ul], axis=0) - x1,y1 = np.where(foot2) - vamax, vamin = np.nanmax(va2), np.nanmin(va2) - vzmax, vzmin = np.nanmax(vz2), np.nanmin(vz2) + vas[x1,y1] = vamin + vzs[x1,y1] = vzmin + x1,y1 = np.where(foot1) + if i == layer.GetFeatureCount()-1: + vamax, vamin = np.nanmax(va1), np.nanmin(va1) + vzmax, vzmin = np.nanmax(vz1), np.nanmin(vz1) if not (p1==p2).all(): - p = np.poly1d(np.polyfit([p1[1], p2[1]],[p1[0], p2[0]],1)) - foot2[x[x > p(y)], y[x > p(y)]] = False - minx, miny = np.where(va2 == vamin) - maxx, maxy = np.where(va2 == vamax) + foot1[x[x <= p(y)], y[x <= p(y)]] = False + minx, miny = np.where(va1 == vamin) + maxx, maxy = np.where(va1 == vamax) min_dst = abs(p(miny*y_scale)-minx*x_scale)/(np.sqrt(1+p.c[0]**2)) max_dst = abs(p(maxy*y_scale)-maxx*x_scale)/(np.sqrt(1+p.c[0]**2)) - if (max_dst < min_dst).any(): - tmp1 = vamin.copy() - vamin = vamax - vamax = tmp1 - minx, miny = np.where(vz2 == vzmin) - maxx, maxy = np.where(vz2 == vzmax) + if (max_dst < min_dst).any(): + tmp1 = vamin.copy() + vamin = vamax + vamax = tmp1 + minx, miny = np.where(vz1 == vzmin) + maxx, maxy = np.where(vz1 == vzmax) min_dst = abs(p(miny*y_scale)-minx*x_scale)/(np.sqrt(1+p.c[0]**2)) max_dst = abs(p(maxy*y_scale)-maxx*x_scale)/(np.sqrt(1+p.c[0]**2)) - if (max_dst < min_dst).any(): - tmp2 = vzmin.copy() - vzmin = vzmax - vzmax = tmp2 + if (max_dst < min_dst).any(): + tmp2 = vzmin.copy() + vzmin = vzmax + vzmax = tmp2 dist = abs(p(y1)-x1)/(np.sqrt(1+p.c[0]**2)) vas[x1,y1] = vamin + dist/(dist.max()-dist.min()) * (vamax-vamin) vzs[x1,y1] = vzmin + dist/(dist.max()-dist.min()) * (vzmax-vzmin) else: - vas[x1,y1] = vamin - vzs[x1,y1] = vzmin - x1,y1 = np.where(foot1) - if i == layer.GetFeatureCount()-1: - vamax, vamin = np.nanmax(va1), np.nanmin(va1) - vzmax, vzmin = np.nanmax(vz1), np.nanmin(vz1) - if not (p1==p2).all(): - foot1[x[x <= p(y)], y[x <= p(y)]] = False - minx, miny = np.where(va1 == vamin) - maxx, maxy = np.where(va1 == vamax) - min_dst = abs(p(miny*y_scale)-minx*x_scale)/(np.sqrt(1+p.c[0]**2)) - max_dst = abs(p(maxy*y_scale)-maxx*x_scale)/(np.sqrt(1+p.c[0]**2)) - if (max_dst < min_dst).any(): - tmp1 = vamin.copy() - vamin = vamax - vamax = tmp1 - minx, miny = np.where(vz1 == vzmin) - maxx, maxy = np.where(vz1 == vzmax) - min_dst = abs(p(miny*y_scale)-minx*x_scale)/(np.sqrt(1+p.c[0]**2)) - max_dst = abs(p(maxy*y_scale)-maxx*x_scale)/(np.sqrt(1+p.c[0]**2)) - if (max_dst < min_dst).any(): - tmp2 = vzmin.copy() - vzmin = vzmax - vzmax = tmp2 - dist = abs(p(y1)-x1)/(np.sqrt(1+p.c[0]**2)) - vas[x1,y1] = vamin + dist/(dist.max()-dist.min()) * (vamax-vamin) - vzs[x1,y1] = vzmin + dist/(dist.max()-dist.min()) * (vzmax-vzmin) - else: - vas[x1,y1] = vamin - vas[x1,y1] = vamin + vas[x1,y1] = vamin + vas[x1,y1] = vamin foot2 = foot1 va2 = va1 vz2 = vz1 - # vas[:] = np.nanmean(vaa.values()) - # vzs[:] = np.nanmean(vza.values()) + # vas[:] = np.nanmean(vaa.values()) + # vzs[:] = np.nanmean(vza.values()) if os.path.exists(view_ang_name): os.remove(view_ang_name) dst_ds = gdal.GetDriverByName('GTiff').Create(view_ang_name, x_size, y_size, 2, gdal.GDT_Int16, options=["TILED=YES", "COMPRESS=DEFLATE"]) @@ -288,18 +277,4 @@ def resample_s2_angles(metafile): src_files = view_ang_names[ret][abs(np.arange(13)[inv_ret][...,None] - np.arange(13)[ret]).argmin(axis=1)] for i in range(len(bad_angs)): copyfile(src_files[i], bad_angs[i]) - #par = partial(get_angle, vaa=vaa, vza=vza, band_dict=band_dict) - #p = Pool(procs) - #ret = p.map(par, view_ang_name_gmls) - #ret =list( map(par, view_ang_name_gmls)) - #p.close() - #p.join() - #ret = np.array(ret) - #if ret.sum()>0: - # bad_angs = view_ang_names[~ret] - # src_files = view_ang_names[ret][abs(np.arange(13)[~ret][...,None] - np.arange(13)[ret]).argmin(axis=1)] - # for i in range(len(bad_angs)): - # copyfile(src_files[i], bad_angs[i]) - #else: - # raise LookupError('failed to reconstract angles...') return sun_ang_name, view_ang_names, toa_refs, cloud_name diff --git a/SIAC/s2_preprocessing.py b/SIAC/s2_preprocessing.py index 814b91e..ff354b1 100644 --- a/SIAC/s2_preprocessing.py +++ b/SIAC/s2_preprocessing.py @@ -3,16 +3,13 @@ import gc import sys import gdal -import logging import numpy as np from glob import glob import multiprocessing from SIAC.reproject import reproject_data from SIAC.s2_angle import resample_s2_angles -from SIAC.create_logger import create_logger from skimage.morphology import disk, binary_dilation, binary_erosion - try: import cPickle as pkl except: @@ -28,11 +25,9 @@ def do_cloud(cloud_bands, cloud_name = None): toas = [reproject_data(str(band), cloud_bands[0], dstNodata=0, resample=5).data for band in cloud_bands] toas = np.array(toas)/10000. mask = np.all(toas >= 0.0001, axis=0) - valid_pixel = mask.sum() cloud_proba = cl.predict_proba(toas[:, mask].T) cloud_mask = np.zeros_like(toas[0]) cloud_mask[mask] = cloud_proba[:,1] - cloud_mask[~mask] = -256 if cloud_name is None: return cloud_mask else: @@ -41,7 +36,6 @@ def do_cloud(cloud_bands, cloud_name = None): dst.SetGeoTransform(g.GetGeoTransform()) dst.SetProjection (g.GetProjection()) dst.GetRasterBand(1).WriteArray((cloud_mask * 100).astype(int)) - dst.GetRasterBand(1).SetNoDataValue(-256) dst=None; g=None return cloud_mask @@ -60,30 +54,9 @@ def s2_pre_processing(s2_dir): aws.append(j) s2_tiles = [] for metafile in scihub + aws: - with open(metafile) as f: - for i in f.readlines(): - if 'TILE_ID' in i: - tile = i.split('')[-1] - log_file = os.path.dirname(metafile) + '/SIAC_S2.log' - if os.path.exists(log_file): - os.remove(log_file) - logger = create_logger(log_file) - logger.propagate = False - logger.info('Preprocessing for %s'%tile) - logger.info('Doing per pixel angle resampling.') sun_ang_name, view_ang_names, toa_refs, cloud_name = resample_s2_angles(metafile) cloud_bands = np.array(toa_refs)[[0,1,3,4,7,8,9,10,11,12]] - logger.info('Getting cloud mask.') - cloud = do_cloud(cloud_bands, cloud_name) - cloud_mask = cloud > 0.6# 0.4 is the sentinel hub default - clean_pixel = ((cloud>=0).sum() - cloud_mask.sum()) / 3348900. * 100. - valid_pixel = (cloud>=0).sum() / 3348900. * 100. - logger.info('Valid pixel percentage: %.02f'%(valid_pixel)) - logger.info('Clean pixel percentage: %.02f'%(clean_pixel)) - cloud_mask = binary_dilation(binary_erosion (cloud_mask, disk(2)), disk(3)) | (cloud < 0) + cloud_mask = do_cloud(cloud_bands, cloud_name) > 0.9 + cloud_mask = binary_dilation(binary_erosion (cloud_mask, disk(2)), disk(3)) s2_tiles.append([sun_ang_name, view_ang_names, toa_refs, cloud_name, cloud_mask, metafile]) - handlers = logger.handlers[:] - for handler in handlers: - handler.close() - logger.removeHandler(handler) return s2_tiles diff --git a/SIAC/tests/MCD43.txt b/SIAC/tests/MCD43.txt deleted file mode 100644 index 7af5944..0000000 --- a/SIAC/tests/MCD43.txt +++ /dev/null @@ -1,33 +0,0 @@ -MCD43A1.A2017311.h18v03.006.2017320031404.hdf -MCD43A1.A2017312.h18v03.006.2017321193550.hdf -MCD43A1.A2017313.h18v03.006.2017322032024.hdf -MCD43A1.A2017314.h18v03.006.2017323031913.hdf -MCD43A1.A2017315.h18v03.006.2017324030906.hdf -MCD43A1.A2017316.h18v03.006.2017325030345.hdf -MCD43A1.A2017317.h18v03.006.2017326031051.hdf -MCD43A1.A2017318.h18v03.006.2017327030935.hdf -MCD43A1.A2017319.h18v03.006.2017328031222.hdf -MCD43A1.A2017320.h18v03.006.2017329032329.hdf -MCD43A1.A2017321.h18v03.006.2017330031421.hdf -MCD43A1.A2017322.h18v03.006.2017331031439.hdf -MCD43A1.A2017323.h18v03.006.2017332031015.hdf -MCD43A1.A2017324.h18v03.006.2017333031645.hdf -MCD43A1.A2017325.h18v03.006.2017334031605.hdf -MCD43A1.A2017326.h18v03.006.2017335032348.hdf -MCD43A1.A2017327.h18v03.006.2017341193157.hdf -MCD43A1.A2017328.h18v03.006.2017341202045.hdf -MCD43A1.A2017329.h18v03.006.2017341204940.hdf -MCD43A1.A2017330.h18v03.006.2017341212126.hdf -MCD43A1.A2017331.h18v03.006.2017341214849.hdf -MCD43A1.A2017332.h18v03.006.2017341220657.hdf -MCD43A1.A2017333.h18v03.006.2017342031106.hdf -MCD43A1.A2017334.h18v03.006.2017345194457.hdf -MCD43A1.A2017335.h18v03.006.2017345202329.hdf -MCD43A1.A2017336.h18v03.006.2017345203903.hdf -MCD43A1.A2017337.h18v03.006.2017346032624.hdf -MCD43A1.A2017338.h18v03.006.2017347025917.hdf -MCD43A1.A2017339.h18v03.006.2017348031721.hdf -MCD43A1.A2017340.h18v03.006.2017349193932.hdf -MCD43A1.A2017341.h18v03.006.2017353224245.hdf -MCD43A1.A2017342.h18v03.006.2017353230736.hdf -MCD43A1.A2017343.h18v03.006.2017353233351.hdf diff --git a/SIAC/tests/test_s2.py b/SIAC/tests/test_s2.py index f62c5a8..e773e92 100644 --- a/SIAC/tests/test_s2.py +++ b/SIAC/tests/test_s2.py @@ -1,10 +1,6 @@ import os, sys -import time import requests -import zipfile import numpy as np -from functools import partial -from multiprocessing import Pool myPath = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, myPath + '/../../') @@ -16,14 +12,19 @@ def test_s2(): from SIAC import SIAC_S2 - from SIAC.downloaders import downloader with open(myPath + '/s2_flists.txt', 'rb') as f: urls = [i.decode().split('\n')[0] for i in f.readlines()] - #headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'} + + import hashlib + def file_as_bytes(file): + with file: + return file.read() + for url in urls: + url = url filename = '/'.join(url.split('/')[8:]) if not os.path.exists(filename): - req = requests.get(url)#, headers=headers) + req = requests.get(url) if req.ok: print('downloading %s' % filename) if not os.path.exists(os.path.dirname(filename)): @@ -34,33 +35,12 @@ def test_s2(): raise with open(filename, "wb") as f: f.write(req.content) - else: - print(req.content) - time.sleep(1) else: pass - #with open(myPath + '/MCD43.txt', 'rb') as f: - # MCD43 = [i.decode().split('\n')[0] for i in f.readlines()] - #if not os.path.exists(os.path.expanduser("~") + '/MCD43/'): - # os.makedirs(os.path.expanduser("~") + '/MCD43/') - #downloader('MCD43.zip', url_root = 'http://www2.geog.ucl.ac.uk/~ucfafyi/mcd43/', file_dir = './') - - #par = partial(downloader, url_root = 'http://www2.geog.ucl.ac.uk/~ucfafyi/mcd43/MCD43/', file_dir = os.path.expanduser("~") + '/MCD43/') - #p = Pool(4) - #p.map(par, MCD43) - #p.close() - #p.join() - #with zipfile.ZipFile("MCD43.zip","r") as zip_ref: - # zip_ref.extractall(os.path.expanduser("~")) - #os.remove("MCD43.zip") - s2_file_dir = filename.split('/')[0] SIAC_S2(s2_file_dir, aoi = str(myPath + '/aoi.geojson')) with open(myPath.replace('tests', '') + 'data/.earthdata_auth', 'wb') as f: f.write(('').encode()) assert True - -if __name__ == '__main__': - test_s2() diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 69fe55e..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 543c6b1..0000000 --- a/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% - -:end -popd diff --git a/docs/source/01_Introduction.rst b/docs/source/01_Introduction.rst deleted file mode 100644 index 462828d..0000000 --- a/docs/source/01_Introduction.rst +++ /dev/null @@ -1,26 +0,0 @@ -SIAC -**************************** - -Introduction -============== - -Land surface reflectance is the fundamental variable for the most of earth observation (EO) missions, -and corrections of the atmospheric disturbs from the cloud, gaseous, aerosol help to get accurate spectral -description of earth surface. Unlike the previous empirical ways of atmospheric correction, we propose a -data fusion method for atmospheric correction of satellite images, with an initial attempt to include the -uncertainty information from different data source. It takes advantage of the high temporal resolution of -MODIS observations to get BRDF description of the earth surface as the prior information of the earth -surface property, uses the ECMWF CAMS Near-real-time as the prior information of the atmospheric sates, -to get optimal estimations of the atmospheric parameters. The code is written in python and we have tested -it with Sentinel 2, Landsat 8, Landsat 5, Sentinel 3 and MODIS data and it shows SIAC can correct the -atmospheric effects reasonablely well. - -Sentinel 2 and Landsat 8 correction examples -============================================ - -A `page `_ shows some correction samples. - - -A `map `_ shows the validation results against AERONET measurements. -If you click points on the scatter plot, it will the comparison between the TOA and BOA refletance and you -can drag to compare between them and click on the image to have spectral comparison over your clicked pixel as well. diff --git a/docs/source/02_Installation.rst b/docs/source/02_Installation.rst deleted file mode 100644 index 120b4fb..0000000 --- a/docs/source/02_Installation.rst +++ /dev/null @@ -1,27 +0,0 @@ -Installation -============= - -1. The standard python way --------------------------- -You can download the source code from the `project website `_. Unpack the file you obtained and then run:: - - python setup.py install - -2. Using pip --------------- -:: - - pip install SIAC - -3. Using anaconda ------------------ -:: - - conda install -c f0xy -c conda-forge siac - -4. From code repository for developing --------------------------------------- - -Installation from the recent stable code repository can be done by:: - - pip install https://github.com/multiply-org/atmospheric_correction/archive/master.zip diff --git a/docs/source/03_Quickstart.rst b/docs/source/03_Quickstart.rst deleted file mode 100644 index 1d880e2..0000000 --- a/docs/source/03_Quickstart.rst +++ /dev/null @@ -1,16 +0,0 @@ -Quickstart -=========== - -The usage of SIAC for Sentinel 2 and Landsat 8 Top Of Atmosphere (TOA) reflectance is very straight forward: - -**Sentinel 2** :: - - from SIAC import SIAC_S2 - SIAC_S2('/directory/where/you/store/S2/data/') # this can be either from AWS or Senitinel offical package - -**Landsat 8** :: - - from SIAC import SIAC_L8 - SIAC_L8('/directory/where/you/store/L8/data/') - -An example usage of SIAC over other sensors and the way to access all the ancillary data is shown with a `jupyter notebook example `_. diff --git a/docs/source/04_Data_access_and_adaption_for_Landsat5.rst b/docs/source/04_Data_access_and_adaption_for_Landsat5.rst deleted file mode 100644 index ed02d31..0000000 --- a/docs/source/04_Data_access_and_adaption_for_Landsat5.rst +++ /dev/null @@ -1,1077 +0,0 @@ -SIAC data accessing and usage on other sensors -===================================================== - -In this Chapter, I will introduce the -`SIAC `__ (Sensor -Invariant Atmospheric Correction) developed under the European Union’s -Horizon 2020 `MULTIPLY `__ project can be -used to generate global **uncertainty quantified analysis ready -datasets** after 2003, which covered by NASA -`Landsat `__ 5-8 missions and ESA `Senitinel -2 `__ -mission. - -SIAC -~~~~ - -This atmospheric correction method uses MODIS MCD43 BRDF product to get -a coarse resolution simulation of earth surface. A model based on MODIS -PSF is built to deal with the scale differences between MODIS and other -sensors, and linear spectral mapping is used to map between different -sensors spectrally. We uses the ECMWF -`CAMS `__ -prediction as a prior for the atmospheric states, coupling with 6S model -to solve for the atmospheric parameters, then the solved atmospheric -parameters are used to correct the TOA reflectances. The whole system is -built under Bayesian theory and the uncertainty is propagated through -the whole system. Since we do not rely on specific bands' relationship -to estimate the atmospheric states, but instead a more generic and -consistent way of inversion those parameters. The code can be downloaded -from `SIAC `__ github -directly and futrher updates will make it more independent and can be -installed on different machines. - -Inputs: -^^^^^^^ - -- `MCD43A1 `__: 16 days before and 16 days after the sensing date -- ECMWF CAMS `Near Real - Time `__ - prediction or MACC - `reanalysis `__: - a time step of 3 hours with the start time of 00:00:00 over the date - and a easier access option is hosted at - http://www2.geog.ucl.ac.uk/~ucfafyi/cams/ but only after 01/04/2015, - when Sentinel 2A was just lucnched. -- Global dem: Global DEM VRT file built from ASTGTM2 DEM, and a bash - script under eles/ can be used to generate with the individual files, - and here we use `ASTER Global Digital Elevation Model - V002 `__ and a easier option - of accessing the dataset with `gdal vertual file - system `__ is - hosted at http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global\_dem.vrt. -- Emulators: emulators for the 6S for different senros, can be found - at: http://www2.geog.ucl.ac.uk/~ucfafyi/emus/ - -Outputs: -^^^^^^^^ - -The outputs are the corrected BOA images saved as ``B0*_sur.tif`` for -each band and uncertainty ``B0*_sur_unc.tif``. ``TOA_RGB.tif`` and -``BOA_RGB.tif`` are generated for a visual check of correction results. -They are all under the same folder as the TOA images. - -Data access: -~~~~~~~~~~~~ - -**MCD43A1** - -The MODIS MCD43A1 Version 6 Bidirectional reflectance distribution -function and Albedo (BRDF/Albedo) Model Parameters data set is a 500 -meter daily 16-day product. The Julian date in the granule ID of each -specific file represents the 9th day of the 16 day retrieval period, and -consequently the observations are weighted to estimate the BRDF/Albedo -for that day. The MCD43A1 algorithm, as is with all combined products, -has the luxury of choosing the best representative pixel from a pool -that includes all the acquisitions from both the Terra and Aqua sensors -from the retrieval period. The MCD43A1 provides the three model -weighting parameters (isotropic, volumetric, and geometric) for each of -the MODIS bands 1 through 7 and the visible (vis), near infrared (nir), -and shortwave bands used to derive the Albedo and BRDF products (MCD43A3 -and MCD43A4). Along with the 3 dimensional parameter layers for these -bands are the Mandatory Quality layers for each of the 10 bands. The -MODIS BRDF/ALBEDO products have achieved stage 3 validation. (From the -website) - -We use the BRDF descriptors inverted from MODIS high temporal -multi-angular observations to get simulation of surface reflectance by -using the Landsat or Sentinel 2 scanning geometry, and the reason of -using 32 days MCD43 is due to the gaps in the current MCD43 products -which cause issues for the inversion of reliable atmospheric paraneters. -This dataset has to be downloaded from the `NASA Data -Pool `__ with username -and passoword registered at `EARTHDATA -LOGIN `__. The function -``get_MCD43.py`` inside ``util`` can be used for a easier acess to the -data, but remember to change the username and passoword in the -``util/earthdata_auth`` file: - -.. code:: - - !cat util/earthdata_auth - - -.. parsed-literal:: - - username - password - - -.. code:: - - import sys - sys.path.insert(0, 'util') - from get_MCD43 import get_mcd43, find_files - from datetime import datetime - # the great gdal virtual file system and google cloud landsat public datasets - google_cloud_base = '/vsicurl/https://storage.googleapis.com/gcp-public-data-landsat/' - aoi = google_cloud_base + 'LE07/01/202/034/LE07_L1TP_202034_20060611_20170108_01_T1/LE07_L1TP_202034_20060611_20170108_01_T1_B1.TIF' - obs_time = datetime(2006, 6, 11) - # based on time and aoi find the MCD43 - # within 16 days temporal window - ret = find_files(aoi, obs_time, temporal_window = 16) - print(ret[0]) - - -.. parsed-literal:: - - https://e4ftl01.cr.usgs.gov/MOTA/MCD43A1.006/2006.05.26/MCD43A1.A2006146.h17v05.006.2016102175833.hdf - - -To download them and used them and creat a daily global VRT file: - -.. code:: - - #get_mcd43(aoi, obs_time, mcd43_dir = './MCD43/', vrt_dir = './MCD43_VRT') - -.. code:: - - ls ./MCD43/ ./MCD43_VRT/ ./MCD43_VRT/2006-05-27/*.vrt - - -.. parsed-literal:: - - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Band_Mandatory_Quality_Band1.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Band_Mandatory_Quality_Band2.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Band_Mandatory_Quality_Band3.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Band_Mandatory_Quality_Band4.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Band_Mandatory_Quality_Band5.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Band_Mandatory_Quality_Band6.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Band_Mandatory_Quality_Band7.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Band_Mandatory_Quality_nir.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Band_Mandatory_Quality_shortwave.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Band_Mandatory_Quality_vis.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Parameters_Band1.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Parameters_Band2.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Parameters_Band3.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Parameters_Band4.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Parameters_Band5.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Parameters_Band6.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Parameters_Band7.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Parameters_nir.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Parameters_shortwave.vrt - ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Parameters_vis.vrt - - ./MCD43/: - 2006_05_26/ - flist.txt - MCD43A1.A2006146.h17v05.006.2016102175833.hdf - MCD43A1.A2006147.h17v05.006.2016102184226.hdf - MCD43A1.A2006148.h17v05.006.2016102192740.hdf - MCD43A1.A2006149.h17v05.006.2016102201522.hdf - MCD43A1.A2006150.h17v05.006.2016102210019.hdf - MCD43A1.A2006151.h17v05.006.2016102215923.hdf - MCD43A1.A2006152.h17v05.006.2016102223051.hdf - MCD43A1.A2006153.h17v05.006.2016102225753.hdf - MCD43A1.A2006154.h17v05.006.2016102232936.hdf - MCD43A1.A2006155.h17v05.006.2016102235934.hdf - MCD43A1.A2006156.h17v05.006.2016103003425.hdf - MCD43A1.A2006157.h17v05.006.2016103010516.hdf - MCD43A1.A2006158.h17v05.006.2016103013644.hdf - MCD43A1.A2006159.h17v05.006.2016103020849.hdf - MCD43A1.A2006160.h17v05.006.2016103023658.hdf - MCD43A1.A2006161.h17v05.006.2016103030549.hdf - MCD43A1.A2006162.h17v05.006.2016103034045.hdf - MCD43A1.A2006163.h17v05.006.2016103041018.hdf - MCD43A1.A2006164.h17v05.006.2016103044452.hdf - MCD43A1.A2006165.h17v05.006.2016103051319.hdf - MCD43A1.A2006166.h17v05.006.2016103054336.hdf - MCD43A1.A2006167.h17v05.006.2016103062221.hdf - MCD43A1.A2006168.h17v05.006.2016103064243.hdf - MCD43A1.A2006169.h17v05.006.2016103123307.hdf - MCD43A1.A2006170.h17v05.006.2016103125809.hdf - MCD43A1.A2006171.h17v05.006.2016103131624.hdf - MCD43A1.A2006172.h17v05.006.2016103133334.hdf - MCD43A1.A2006173.h17v05.006.2016103134900.hdf - MCD43A1.A2006174.h17v05.006.2016103140345.hdf - MCD43A1.A2006175.h17v05.006.2016103141904.hdf - MCD43A1.A2006176.h17v05.006.2016103143227.hdf - MCD43A1.A2006177.h17v05.006.2016103144345.hdf - MCD43A1.A2006178.h17v05.006.2016103150133.hdf - MCD43A1.A2011232.h17v05.006.2016135180015.hdf - MCD43A1.A2011233.h17v05.006.2016135180956.hdf - MCD43A1.A2011234.h17v05.006.2016135182004.hdf - MCD43A1.A2011235.h17v05.006.2016135183413.hdf - MCD43A1.A2011236.h17v05.006.2016135184405.hdf - MCD43A1.A2011237.h17v05.006.2016135185631.hdf - MCD43A1.A2011238.h17v05.006.2016135190844.hdf - MCD43A1.A2011239.h17v05.006.2016135192024.hdf - MCD43A1.A2011240.h17v05.006.2016135193355.hdf - MCD43A1.A2011241.h17v05.006.2016135194900.hdf - MCD43A1.A2011242.h17v05.006.2016135200116.hdf - MCD43A1.A2011243.h17v05.006.2016135201359.hdf - MCD43A1.A2011244.h17v05.006.2016135202542.hdf - MCD43A1.A2011245.h17v05.006.2016135203726.hdf - MCD43A1.A2011246.h17v05.006.2016135204754.hdf - MCD43A1.A2011247.h17v05.006.2016135210022.hdf - MCD43A1.A2011248.h17v05.006.2016135211314.hdf - MCD43A1.A2011249.h17v05.006.2016135212256.hdf - MCD43A1.A2011250.h17v05.006.2016137122909.hdf - MCD43A1.A2011251.h17v05.006.2016137125638.hdf - MCD43A1.A2011252.h17v05.006.2016137131949.hdf - MCD43A1.A2011253.h17v05.006.2016137134319.hdf - MCD43A1.A2011254.h17v05.006.2016137140557.hdf - MCD43A1.A2011255.h17v05.006.2016137143341.hdf - MCD43A1.A2011256.h17v05.006.2016137144846.hdf - MCD43A1.A2011257.h17v05.006.2016137151738.hdf - MCD43A1.A2011258.h17v05.006.2016137153854.hdf - MCD43A1.A2011259.h17v05.006.2016137155602.hdf - MCD43A1.A2011260.h17v05.006.2016137161759.hdf - MCD43A1.A2011261.h17v05.006.2016137164021.hdf - MCD43A1.A2011262.h17v05.006.2016137170721.hdf - MCD43A1.A2011263.h17v05.006.2016137172812.hdf - MCD43A1.A2011264.h17v05.006.2016137175047.hdf - - ./MCD43_VRT/: - 2006-05-26/ 2006-06-06/ 2006-06-17/ 2011-08-20/ 2011-08-31/ 2011-09-11/ - 2006-05-27/ 2006-06-07/ 2006-06-18/ 2011-08-21/ 2011-09-01/ 2011-09-12/ - 2006-05-28/ 2006-06-08/ 2006-06-19/ 2011-08-22/ 2011-09-02/ 2011-09-13/ - 2006-05-29/ 2006-06-09/ 2006-06-20/ 2011-08-23/ 2011-09-03/ 2011-09-14/ - 2006-05-30/ 2006-06-10/ 2006-06-21/ 2011-08-24/ 2011-09-04/ 2011-09-15/ - 2006-05-31/ 2006-06-11/ 2006-06-22/ 2011-08-25/ 2011-09-05/ 2011-09-16/ - 2006-06-01/ 2006-06-12/ 2006-06-23/ 2011-08-26/ 2011-09-06/ 2011-09-17/ - 2006-06-02/ 2006-06-13/ 2006-06-24/ 2011-08-27/ 2011-09-07/ 2011-09-18/ - 2006-06-03/ 2006-06-14/ 2006-06-25/ 2011-08-28/ 2011-09-08/ 2011-09-19/ - 2006-06-04/ 2006-06-15/ 2006-06-26/ 2011-08-29/ 2011-09-09/ 2011-09-20/ - 2006-06-05/ 2006-06-16/ 2006-06-27/ 2011-08-30/ 2011-09-10/ 2011-09-21/ - - -.. code:: - - !gdalinfo ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Parameters_Band5.vrt - - -.. parsed-literal:: - - Driver: VRT/Virtual Raster - Files: ./MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Parameters_Band5.vrt - Size is 2400, 2400 - Coordinate System is: - PROJCS["unnamed", - GEOGCS["Unknown datum based upon the custom spheroid", - DATUM["Not specified (based on custom spheroid)", - SPHEROID["Custom spheroid",6371007.181,0]], - PRIMEM["Greenwich",0], - UNIT["degree",0.0174532925199433]], - PROJECTION["Sinusoidal"], - PARAMETER["longitude_of_center",0], - PARAMETER["false_easting",0], - PARAMETER["false_northing",0], - UNIT["Meter",1]] - Origin = (-1111950.519667000044137,4447802.078666999936104) - Pixel Size = (463.312716527916677,-463.312716527916677) - Corner Coordinates: - Upper Left (-1111950.520, 4447802.079) ( 13d 3'14.66"W, 40d 0' 0.00"N) - Lower Left (-1111950.520, 3335851.559) ( 11d32'49.22"W, 30d 0' 0.00"N) - Upper Right ( 0.000, 4447802.079) ( 0d 0' 0.01"E, 40d 0' 0.00"N) - Lower Right ( 0.000, 3335851.559) ( 0d 0' 0.01"E, 30d 0' 0.00"N) - Center ( -555975.260, 3891826.819) ( 6d 6'13.94"W, 35d 0' 0.00"N) - Band 1 Block=128x128 Type=Int16, ColorInterp=Gray - NoData Value=32767 - Band 2 Block=128x128 Type=Int16, ColorInterp=Gray - NoData Value=32767 - Band 3 Block=128x128 Type=Int16, ColorInterp=Gray - NoData Value=32767 - - -The great part of creating VRT files is that virtual global mosaic of -MCD43 for different parameters and different times are created with a -very small fraction of storage space, but eliminate a lot of troubles in -dealing with different spatial resolutions, data formats and multi-tile -coverage... Actually, the best way of storing the datasets is turn it -into `Cloud optimized -GeoTIFF `__, -which enables access of chuncks of data from a virtul mosaic to be -possible and saves a lot of unnecessary downloading of data outside the -area of interest. And this can be done easily with gdal as well: - -.. code:: - - #!/usr/bin/env python - import os - import sys - import gdal - from datetime import datetime - fname = './MCD43/MCD43A1.A2006146.h17v05.006.2016102175833.hdf' - try: - g = gdal.Open(fname) - except: - raise IOError('File cannot opened!') - subs = [i[0] for i in g.GetSubDatasets()] - - def translate(sub): - ret = sub.split(':') - path, para = ret[2].split('"')[1], ret[-1] - base = '/'.join(path.split('/')[:-1]) - fname = path.split('/')[-1] - day = datetime.strptime(fname.split('.')[1].split('A')[1], '%Y%j').strftime('%Y_%m_%d') - ret = fname.split('.') - day = datetime.strptime(ret[-5].split('A')[1], '%Y%j').strftime('%Y_%m_%d') - fname = '_'.join(ret[:-2]) + '_%s.tif'%para - fname = base + '/' + day + '/' + fname - if os.path.exists(fname): - pass - else: - if os.path.exists(base + '/%s'%day): - pass - else: - os.mkdir(base + '/%s'%day) - gdal.Translate(fname, sub,creationOptions=["TILED=YES", "COMPRESS=DEFLATE"]) - for sub in subs: - translate(sub) - -.. code:: - - ls MCD43/2006_05_26/ - - -.. parsed-literal:: - - flist.txt - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Band_Mandatory_Quality_Band1.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Band_Mandatory_Quality_Band2.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Band_Mandatory_Quality_Band3.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Band_Mandatory_Quality_Band4.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Band_Mandatory_Quality_Band5.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Band_Mandatory_Quality_Band6.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Band_Mandatory_Quality_Band7.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Band_Mandatory_Quality_nir.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Band_Mandatory_Quality_shortwave.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Band_Mandatory_Quality_vis.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_Band1.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_Band1.tif.aux.xml - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_Band2.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_Band2.tif.aux.xml - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_Band3.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_Band3.tif.aux.xml - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_Band4.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_Band4.tif.aux.xml - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_Band5.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_Band5.tif.aux.xml - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_Band6.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_Band6.tif.aux.xml - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_Band7.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_Band7.tif.aux.xml - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_nir.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_nir.tif.aux.xml - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_shortwave.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_shortwave.tif.aux.xml - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_vis.tif - MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_vis.tif.aux.xml - - -So we basically converted the MODIS HDF format into GeoTiff format and -an important argument for the ``gdal.translate`` is ``TILED=YES`` which -will make small chunk access of dataset to be possible. - -.. code:: - - #!/usr/bin/env python - import pylab as plt - import numpy as np - from reproject import reproject_data - # hear we try to reproject the RGB band - # from MCD43 to the aoi used above, whcih - # is just a url to the google cloud file - # we also use the vrt file we created - # as our source file and reproject them - # to the aoi with the same spatial resol. - source = './MCD43_VRT/2006-05-27/MCD43_2006147_BRDF_Albedo_Parameters_Band1.vrt' - - r = reproject_data(source, aoi).data[0] * 0.001 - g = reproject_data(source.replace('Band1', 'Band4'), aoi).data[0] * 0.001 - b = reproject_data(source.replace('Band1', 'Band3'), aoi).data[0] * 0.001 - - r[r>1] = np.nan - b[b>1] = np.nan - g[g>1] = np.nan - plt.imshow(np.array([r,g,b]).transpose(1,2,0)*4) - - -.. parsed-literal:: - - - - - -.. raw:: html - - - - - - -.. parsed-literal:: - - - - - -I have also put the created tif file into the UCL geography file server -at http://www2.geog.ucl.ac.uk/~ucfafyi/test\_files/2006\_05\_26/ - -.. code:: - - from IPython.core.display import HTML - #HTML("http://www2.geog.ucl.ac.uk/~ucfafyi/test_files/2006_05_26/") - -And if we change from the VRT file to the url to the tif files, it will -also works!!! - -.. code:: - - - url = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/test_files/2006_05_26/' - source = url + 'MCD43A1_A2006146_h17v05_006_BRDF_Albedo_Parameters_Band1.tif' - - r = reproject_data(source, aoi).data[0] * 0.001 - g = reproject_data(source.replace('Band1', 'Band4'), aoi).data[0] * 0.001 - b = reproject_data(source.replace('Band1', 'Band3'), aoi).data[0] * 0.001 - - r[r>1]=np.nan - b[b>1] = np.nan - g[g>1] = np.nan - plt.imshow(np.array([r,g,b]).transpose(1,2,0)*4) - - - -.. parsed-literal:: - - - - - -.. raw:: html - - - - - - -.. parsed-literal:: - - - - - -And if we creat virtual global mosaic VRT file with those GeoTiff -images, we can also access them with gdal and do the subset and -reprojection easily....And I will demonstrate it with the ASTGTM2 DEM... - -**Global DEM** - -I have downloaded most of the DEM images from NASA server and put them -in the UCL server at: http://www2.geog.ucl.ac.uk/~ucfafyi/eles/ and a -global DEM VRT file is generated with: - -.. code:: bash - - ls *.tif>file_list.txt - gdalbuildvrt -te -180 -90 180 90 global_dem.vrt -input_file_list file_list.txt - -.. code:: - - print(gdal.Info('/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt',showFileList=False)) - - -.. parsed-literal:: - - Driver: VRT/Virtual Raster - Files: /vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt - Size is 1296000, 648000 - Coordinate System is: - GEOGCS["WGS 84", - DATUM["WGS_1984", - SPHEROID["WGS 84",6378137,298.257223563, - AUTHORITY["EPSG","7030"]], - AUTHORITY["EPSG","6326"]], - PRIMEM["Greenwich",0], - UNIT["degree",0.0174532925199433], - AUTHORITY["EPSG","4326"]] - Origin = (-180.000000000000000,90.000000000000000) - Pixel Size = (0.000277777777778,-0.000277777777778) - Corner Coordinates: - Upper Left (-180.0000000, 90.0000000) (180d 0' 0.00"W, 90d 0' 0.00"N) - Lower Left (-180.0000000, -90.0000000) (180d 0' 0.00"W, 90d 0' 0.00"S) - Upper Right ( 180.0000000, 90.0000000) (180d 0' 0.00"E, 90d 0' 0.00"N) - Lower Right ( 180.0000000, -90.0000000) (180d 0' 0.00"E, 90d 0' 0.00"S) - Center ( 0.0000000, -0.0000000) ( 0d 0' 0.00"E, 0d 0' 0.00"S) - Band 1 Block=128x128 Type=Int16, ColorInterp=Gray - - - -.. code:: - - - source = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/eles/global_dem.vrt' - ele = reproject_data(source, aoi).data * 0.001 - plt.imshow(ele) - - - -.. parsed-literal:: - - - - - -.. raw:: html - - - - - - -.. parsed-literal:: - - - - - -Instantly, we get the DEM with the same resolution and geographic -coverage as the aoi, which means if we have the MCD43 in GeoTiff format -and a global mosaic can be created for each day then the access of MCD43 -data will be much easier, and this actully applies to all different kind -of GIS datasets. - -**CAMS atmospheric composition data** - -As part of the European Copernicus programme on environmental -monitoring, greenhouse gases, aerosols, and chemical species have been -introduced in the ECMWF model allowing assimilation and forecasting of -atmospheric composition. At the same time, the added atmospheric -composition variables are being used to improve the Numerical Weather -Prediction (NWP) system itself, most notably through the interaction -with the radiation scheme and the use in observation operators for -satellite radiance assimilation. (from the -`website `__) - -In SIAC, the aerosol optical thickness (AOT) at 550\ :math:`nm`, total -column of water vapour (TCWV) and total column of Ozone(TCO\ :math:`_3`) -are used as priors for the atmospheric states. The data can be aquired -through the official pages, but it needs to wait for the queue to -process each time you request it, but actually the dataset is at a -coarse grid and only take small storage space and again daily global -mosaic for each parameter in tif format at UCL server at -http://www2.geog.ucl.ac.uk/~ucfafyi/cams/. There is no need to process -it for each user and do the subset each time.... - -The api access to cams near real time: - -.. code:: python - - #!/usr/bin/env python - import os - import sys - import gdal - from glob import glob - from ecmwfapi import ECMWFDataServer - server = ECMWFDataServer() - from datetime import datetime, timedelta - para_names = 'tcwv,gtco3,aod550,duaod550,omaod550,bcaod550,suaod550'.split(',') - this_date = datetime(2015,4,26) - filename = "%s.nc"%this_date - if not os.path.exists(filename): - server.retrieve({ - "class": "mc", - "dataset": "cams_nrealtime", - "date": "%s"%this_date, - "expver": "0001", - "levtype": "sfc", - "param": "137.128/206.210/207.210/209.210/210.210/211.210/212.210", - "step": "0/3/6/9/12/15/18/21/24", - "stream": "oper", - "time": "00:00:00", - "type": "fc", - "grid": "0.75/0.75", - "area": "90/-180/-90/180", - "format":"netcdf", - "target": "%s.nc"%this_date, - }) - else: - pass - header = '_'.join(filename.split('.')[0].split('-')) - if not os.path.exists(header): - os.mkdir(header) - exists = glob(header+'/*.tif') - if len(sys.argv[2:])>0: - list_para = sys.argv[2:] - else: - list_para = para_names - temp = 'NETCDF:"%s":%s' - for i in list_para: - if header + '/'+header+'_'+i+'.tif' not in exists: - t = 'Translating %-31s to %-23s'%(temp%(filename,i), header+'_'+i+'.tif') - print(t) - gdal.Translate(header + '/'+header+'_'+i+'.tif', temp%(filename,i), outputSRS='EPSG:4326', creationOptions=["TILED=YES", "COMPRESS=DEFLATE"]) - -and reanalysis data: - -.. code:: python - - #!/usr/bin/env python - import os - import sys - import gdal - from glob import glob - from ecmwfapi import ECMWFDataServer - server = ECMWFDataServer() - from datetime import datetime, timedelta - para_names = 'tcwv,gtco3,aod550,duaod550,omaod550,bcaod550,suaod550'.split(',') - this_date = datetime(2012,4,26) - filename = "%s.nc"%this_date - if not os.path.exists(filename): - server.retrieve({ - "class": "mc", - "dataset": "macc", - "date": "%s"%this_date, - "expver": "rean", - "levtype": "sfc", - "param": "137.128/206.210/207.210/209.210/210.210/211.210/212.210", - "step": "0/3/6/9/12/15/18/21/24", - "stream": "oper", - "time": "00:00:00", - "type": "fc", - "grid": "0.75/0.75", - "area": "90/-180/-90/180", - "format":"netcdf", - "target": "%s.nc"%this_date, - }) - else: - pass - header = '_'.join(filename.split('.')[0].split('-')) - if not os.path.exists(header): - os.mkdir(header) - exists = glob(header+'/*.tif') - if len(sys.argv[2:])>0: - list_para = sys.argv[2:] - else: - list_para = para_names - temp = 'NETCDF:"%s":%s' - for i in list_para: - if header + '/'+header+'_'+i+'.tif' not in exists: - t = 'Translating %-31s to %-23s'%(temp%(filename,i), header+'_'+i+'.tif') - print(t) - gdal.Translate(header + '/'+header+'_'+i+'.tif', temp%(filename,i), outputSRS='EPSG:4326', creationOptions=["TILED=YES", "COMPRESS=DEFLATE"]) - -.. code:: - - - # here we test with subset of global AOT 550 - # over the aoi - source = '/vsicurl/http://www2.geog.ucl.ac.uk/~ucfafyi/cams/2015_09_08/2015_09_08_aod550.tif' - g = gdal.Open(source) - b1 = g.GetRasterBand(1) - scale, offset = b1.GetScale(), b1.GetOffset() - g = None - g = reproject_data(source, aoi).g - aot = scale * g.GetRasterBand(3).ReadAsArray() + offset - plt.imshow(aot) - - - -.. parsed-literal:: - - - - - -.. raw:: html - - - - - - -.. parsed-literal:: - - - - - -**TOA reflectance** - -Unlike many AC method, one needs to convert reflectance to radiance and -back to reflectance after AC, SIAC only need the reflectance ranging -from 0-1 (or 10, 100, 1000... but you need to give the scale and offset -vlaues). To access the Landsat datasets, you can download it from -`USGS `__ or google has mirrored the -whole Landsat datesets which can be accessed from the google public -`Landsat -dateset `__ -with url: ``https://storage.googleapis.com/gcp-public-data-landsat/``. - -Here we show example of accessing different Landsat mission images, with -the -`AOI `__ -in Spain, and we stored it as a Geojson file. - -.. code:: - - import requests - # Landsat 5 - base = 'https://storage.googleapis.com/gcp-public-data-landsat/' - tile = 'LT05/01/202/034/LT05_L1TP_202034_20110905_20161006_01_T1/' - bands = ['LT05_L1TP_202034_20110905_20161006_01_T1_B%d.TIF'%i for i in [1,2,3,4,5,7]] - metadata = requests.get(base + tile + 'LT05_L1TP_202034_20110905_20161006_01_T1_MTL.txt').content.decode() - scale = [] - off = [] - for i in metadata.split('\n'): - if 'REFLECTANCE_ADD_BAND' in i: - print(i) - off.append(float(i.split(' = ')[1])) - if 'REFLECTANCE_MULT_BAND' in i: - print(i) - scale.append(float(i.split(' = ')[1])) - rgb = [] - for i in [0,1,2]: - g = gdal.Warp('', '/vsicurl/' + base + tile + bands[i], format = 'MEM', warpOptions = \ - ['NUM_THREADS=ALL_CPUS'],srcNodata = 0, dstNodata=0, cutlineDSName= 'aoi.json', \ - cropToCutline=True, resampleAlg = gdal.GRIORA_NearestNeighbour) - rgb.append(g.ReadAsArray()) - rgb = np.array(scale[:3])[...,None, None]*rgb + np.array(off[:3])[...,None, None] - alpha = np.any(rgb < 0, axis=0) - - -.. parsed-literal:: - - REFLECTANCE_MULT_BAND_1 = 1.2582E-03 - REFLECTANCE_MULT_BAND_2 = 2.6296E-03 - REFLECTANCE_MULT_BAND_3 = 2.2379E-03 - REFLECTANCE_MULT_BAND_4 = 2.7086E-03 - REFLECTANCE_MULT_BAND_5 = 1.8340E-03 - REFLECTANCE_MULT_BAND_7 = 2.5458E-03 - REFLECTANCE_ADD_BAND_1 = -0.003756 - REFLECTANCE_ADD_BAND_2 = -0.007786 - REFLECTANCE_ADD_BAND_3 = -0.004746 - REFLECTANCE_ADD_BAND_4 = -0.007377 - REFLECTANCE_ADD_BAND_5 = -0.007472 - REFLECTANCE_ADD_BAND_7 = -0.008371 - - -.. code:: - - # since landsat angles has to be produced - # from the angular text file - ang = requests.get(base + tile + 'LT05_L1TP_202034_20110905_20161006_01_T1_ANG.txt').content - with open('landsat/landsat_ang/LT05_L1TP_202034_20110905_20161006_01_T1_ANG.txt', 'wb') as f: - f.write(ang) - - header = 'LT05_L1TP_202034_20110905_20161006_01_T1_' - import os - from glob import glob - from multiprocessing import Pool - import subprocess - from functools import partial - cwd = os.getcwd() - #header += cwd - os.chdir('landsat/landsat_ang/') - def translate_angle(band, header): - subprocess.call([cwd+'/util/landsat_angles/landsat_angles', \ - header + 'ANG.txt', '-B', '%d'%band]) - inp = 'angle_sensor_B%02d.img'%band - oup = header+ 'VZA_VAA_B%02d.TIF'%band - if os.path.exists(oup): - os.remove(oup) - gdal.Translate(oup, inp, creationOptions = \ - ['COMPRESS=DEFLATE', 'TILED=YES'], noData='-32767').FlushCache() - if not os.path.exists(header + 'SZA_SAA.TIF'): - gdal.Translate(header + 'SZA_SAA.TIF', 'angle_solar_B01.img', creationOptions = \ - ['COMPRESS=DEFLATE', 'TILED=YES'], noData='-32767').FlushCache() - p = Pool() - par = partial(translate_angle, header=header) - p.map(par, range(1,8)) - list(map(os.remove, glob('angle_s*_B*.img*'))) - os.chdir(cwd) - -.. code:: - - sza, saa = gdal.Warp('', 'landsat/landsat_ang/'+ header + 'SZA_SAA.TIF',format = 'MEM', warpOptions = \ - ['NUM_THREADS=ALL_CPUS'],srcNodata = 0, dstNodata=0, cutlineDSName= 'aoi.json', \ - cropToCutline=True, resampleAlg = gdal.GRIORA_NearestNeighbour).ReadAsArray()/100. - rgb = rgb / np.cos(np.deg2rad(sza)) - rgb[rgb<0] = np.nan - rgba = np.r_[rgb, ~alpha[None, ...]] - -.. code:: - - plt.figure(figsize=(8,8)) - plt.imshow(rgba.transpose(1,2,0)*2) - - - -.. parsed-literal:: - - - - - -.. raw:: html - - - - - - -.. parsed-literal:: - - - - - -.. code:: - - # We save all the remote file to local - # also convert it to reflectance... - - for i in range(6): - g = gdal.Warp('', '/vsicurl/' + base + tile + bands[i], format = 'MEM', warpOptions = \ - ['NUM_THREADS=ALL_CPUS'],srcNodata = 0, dstNodata=0, cutlineDSName= 'aoi.json', \ - cropToCutline=True, resampleAlg = gdal.GRIORA_NearestNeighbour) - data = (g.ReadAsArray() * scale[i] + off[i])/np.cos(np.deg2rad(sza)) - data[data<0] = -9999 - driver = gdal.GetDriverByName('GTiff') - ds = driver.Create('landsat/' + bands[i], data.shape[1], data.shape[0], 1, \ - gdal.GDT_Float32, options=["TILED=YES", "COMPRESS=DEFLATE"]) - ds.SetGeoTransform(g.GetGeoTransform()) - ds.SetProjection(g.GetProjectionRef()) - ds.GetRasterBand(1).WriteArray(data) - ds.FlushCache() - ds=None - g = gdal.Warp('landsat/' + bands[0].replace('B1', 'BQA'), '/vsicurl/' + base + tile + bands[0].replace('B1', 'BQA'), format = 'GTiff', warpOptions = \ - ['NUM_THREADS=ALL_CPUS'],srcNodata = 0, dstNodata=0, cutlineDSName= 'aoi.json', \ - cropToCutline=True, resampleAlg = gdal.GRIORA_NearestNeighbour, creationOptions \ - = ['COMPRESS=DEFLATE', 'TILED=YES']) - g.FlushCache() - aoi_mask = np.isnan(data) - -.. code:: - - - plt.imshow(gdal.Open('landsat/LT05_L1TP_202034_20110905_20161006_01_T1_B1.TIF').ReadAsArray()) - plt.colorbar() - - - -.. parsed-literal:: - - - - - -.. raw:: html - - - - - - -.. parsed-literal:: - - - - - -We now have the angle and reflectance for this AOI, we can pass them -into SIAC to run the atmospheric correction. - -Run SIAC -~~~~~~~~ - -In reality, we need the emulators for this sensor, but at the moment we -do not have, so we instead use Landsat 8 emultors just for demostration -purporse. - -.. code:: - - import numpy as np - from SIAC.the_aerosol import solve_aerosol - from SIAC.the_correction import atmospheric_correction - sensor_sat = 'OLI', 'L8' - toa_bands = ['landsat/'+i for i in bands] - view_angles = ['landsat/landsat_ang/LT05_L1TP_202034_20110905_20161006_01_T1_VZA_VAA_%s.TIF'%i \ - for i in ['B01', 'B02', 'B03', 'B04', 'B05', 'B07']] - sun_angles = 'landsat/landsat_ang/LT05_L1TP_202034_20110905_20161006_01_T1_SZA_SAA.TIF' - cloud_mask = gdal.Open('landsat/' + bands[0].replace('B1', 'BQA')).ReadAsArray() - cloud_mask = ~((cloud_mask >= 672) & ( cloud_mask <= 684)) | aoi_mask - band_wv = [469, 555, 645, 859, 1640, 2130] - obs_time = datetime(2011, 9, 5, 10, 51, 11) - #get_mcd43(toa_bands[0], obs_time, mcd43_dir = './MCD43/', vrt_dir = './MCD43_VRT') - band_index = [1,2,3,4,5,6] - aero = solve_aerosol(sensor_sat,toa_bands,band_wv, band_index,\ - view_angles,sun_angles,obs_time,cloud_mask,aot_prior = 0.05, \ - aot_unc=0.1, mcd43_dir= './MCD43_VRT/',gamma=10., ref_scale = 1., ref_off = 0., a_z_order=0) - aero._solving() - - base = 'landsat/LT05_L1TP_202034_20110905_20161006_01_T1_' - aot = base + 'aot.tif' - tcwv = base + 'tcwv.tif' - tco3 = base + 'tco3.tif' - aot_unc = base + 'aot_unc.tif' - tcwv_unc = base + 'tcwv_unc.tif' - tco3_unc = base + 'tco3_unc.tif' - rgb = [toa_bands[2], toa_bands[1], toa_bands[0]] - atmo = atmospheric_correction(sensor_sat, toa_bands, band_index,view_angles,sun_angles, \ - aot = aot, cloud_mask = cloud_mask, tcwv = tcwv, tco3 = tco3, \ - aot_unc = aot_unc, tcwv_unc = tcwv_unc, tco3_unc = tco3_unc, \ - rgb = rgb, ref_scale = 1., ref_off = 0., a_z_order = 0) - ret = atmo._doing_correction() - - -.. parsed-literal:: - - 2018-09-13 18:41:35,579 - AtmoCor - INFO - Set AOI. - 2018-09-13 18:41:35,581 - AtmoCor - INFO - Get corresponding bands. - 2018-09-13 18:41:35,581 - AtmoCor - INFO - Slice TOA bands based on AOI. - 2018-09-13 18:41:36,815 - AtmoCor - INFO - Parsing angles. - 2018-09-13 18:41:36,821 - AtmoCor - INFO - Mask bad pixeles. - 2018-09-13 18:41:37,122 - AtmoCor - INFO - Get simulated BOA. - 2018-09-13 18:42:25,089 - AtmoCor - INFO - Get PSF. - 2018-09-13 18:42:28,284 - AtmoCor - INFO - Solved PSF: 8.67, 11.33, 0, 4, 1, and R value is: 0.961. - 2018-09-13 18:42:28,288 - AtmoCor - INFO - Get simulated TOA reflectance. - 2018-09-13 18:42:31,141 - AtmoCor - INFO - Filtering data. - 2018-09-13 18:42:31,199 - AtmoCor - INFO - Loading emulators. - 2018-09-13 18:42:33,766 - AtmoCor - INFO - Reading priors and elevation. - 2018-09-13 18:42:42,059 - MultiGrid solver - INFO - MultiGrid solver in process... - 2018-09-13 18:42:42,060 - MultiGrid solver - INFO - Total 5 level of grids are going to be used. - 2018-09-13 18:42:42,061 - MultiGrid solver - INFO - [94mOptimizing at grid level 1 - - -.. parsed-literal:: - - +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - - -.. parsed-literal:: - - 2018-09-13 18:42:43,462 - MultiGrid solver - INFO - [92mb'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH' - 2018-09-13 18:42:43,463 - MultiGrid solver - INFO - [92mIterations: 4 - 2018-09-13 18:42:43,463 - MultiGrid solver - INFO - [92mFunction calls: 23 - 2018-09-13 18:42:43,464 - MultiGrid solver - INFO - [94mOptimizing at grid level 2 - - -.. parsed-literal:: - - +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - - -.. parsed-literal:: - - 2018-09-13 18:42:46,115 - MultiGrid solver - INFO - [92mb'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH' - 2018-09-13 18:42:46,116 - MultiGrid solver - INFO - [92mIterations: 3 - 2018-09-13 18:42:46,116 - MultiGrid solver - INFO - [92mFunction calls: 25 - 2018-09-13 18:42:46,117 - MultiGrid solver - INFO - [94mOptimizing at grid level 3 - - -.. parsed-literal:: - - +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - - -.. parsed-literal:: - - 2018-09-13 18:42:57,283 - MultiGrid solver - INFO - [92mb'STOP: TOTAL NO. of f AND g EVALUATIONS EXCEEDS LIMIT' - 2018-09-13 18:42:57,283 - MultiGrid solver - INFO - [92mIterations: 8 - 2018-09-13 18:42:57,284 - MultiGrid solver - INFO - [92mFunction calls: 75 - 2018-09-13 18:42:57,284 - MultiGrid solver - INFO - [94mOptimizing at grid level 4 - - -.. parsed-literal:: - - +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - - -.. parsed-literal:: - - 2018-09-13 18:43:09,654 - MultiGrid solver - INFO - [92mb'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH' - 2018-09-13 18:43:09,655 - MultiGrid solver - INFO - [92mIterations: 10 - 2018-09-13 18:43:09,655 - MultiGrid solver - INFO - [92mFunction calls: 36 - 2018-09-13 18:43:09,656 - MultiGrid solver - INFO - [94mOptimizing at grid level 5 - - -.. parsed-literal:: - - +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - - -.. parsed-literal:: - - 2018-09-13 18:45:07,040 - MultiGrid solver - INFO - [92mb'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH' - 2018-09-13 18:45:07,040 - MultiGrid solver - INFO - [92mIterations: 10 - 2018-09-13 18:45:07,041 - MultiGrid solver - INFO - [92mFunction calls: 32 - 2018-09-13 18:45:10,977 - AtmoCor - INFO - Finished retrieval and saving them into local files. - 2018-09-13 18:45:13,309 - AtmoCor - INFO - Set AOI. - 2018-09-13 18:45:13,310 - AtmoCor - INFO - Slice TOA bands based on AOI. - 2018-09-13 18:45:14,708 - AtmoCor - INFO - Parsing angles. - 2018-09-13 18:45:15,728 - AtmoCor - INFO - Parsing auxs. - 2018-09-13 18:45:16,584 - AtmoCor - INFO - Parsing atmo parameters. - 2018-09-13 18:45:23,338 - AtmoCor - INFO - Loading emus. - 2018-09-13 18:45:25,858 - AtmoCor - INFO - Get correction coefficients. - 2018-09-13 18:45:32,039 - AtmoCor - INFO - Doing corrections. - 2018-09-13 18:45:37,474 - AtmoCor - INFO - Composing RGB. - 2018-09-13 18:45:39,423 - AtmoCor - INFO - Done. - - - - -+-------------+-------------+ -| TOA image | BOA image | -+=============+=============+ -| |image2| | |image3| | -+-------------+-------------+ - -.. |image0| image:: https://github.com/MarcYin/Global-analysis-ready-dataset/blob/master/landsat/TOA_overview.png?raw=true -.. |image1| image:: https://github.com/MarcYin/Global-analysis-ready-dataset/blob/master/landsat/BOA_overview.png?raw=true -.. |image2| image:: https://github.com/MarcYin/Global-analysis-ready-dataset/blob/master/landsat/TOA_overview.png?raw=true -.. |image3| image:: https://github.com/MarcYin/Global-analysis-ready-dataset/blob/master/landsat/BOA_overview.png?raw=true - -.. code:: - - - toa = [] - boa = [] - for i in range(6): - toa.append(gdal.Open(toa_bands[i]).ReadAsArray(1500, 500, 1,1)[0,0]) - boa.append((gdal.Open(toa_bands[i].replace('.TIF', '_sur.tif')).ReadAsArray(1500, 500, 1,1)[0,0])/10000.) - plt.plot(band_wv, toa, 's-', label='TOA') - plt.plot(band_wv, np.array(boa), 'o-', label='BOA') - plt.legend() - - - -.. parsed-literal:: - - - - - -.. raw:: html - - - - - - -.. parsed-literal:: - - - - - -Here we demonstrate the use of SIAC for the correction of Landsat 5 -images, but it can only treated as a test and if one wants to do the -real AC of Landsat 5 collection, the emulators and spectral mapping -should be created for Landsat 5 TM sensor also a more resonable prior -should be used. - - diff --git a/docs/source/05_Annex_technical.rst b/docs/source/05_Annex_technical.rst deleted file mode 100644 index bdb4d53..0000000 --- a/docs/source/05_Annex_technical.rst +++ /dev/null @@ -1,48 +0,0 @@ -Technical documentation -======================= - -`SIAC_S2` --------------- - -.. automodule:: SIAC.SIAC_S2 - :members: SIAC_S2 - :noindex: - :undoc-members: - :show-inheritance: - -`SIAC_L8` --------------- - -.. automodule:: SIAC.SIAC_L8 - :members: SIAC_L8 - :noindex: - :undoc-members: - :show-inheritance: - -`Atmospheric parameters Inversion` ----------------------------------- - -.. automodule:: SIAC.the_aerosol - :members: solve_aerosol - :noindex: - :undoc-members: - :show-inheritance: - -`Atmospheric parameters Solver` -------------------------------- - -.. automodule:: SIAC.atmo_solver - :members: solving_atmo_paras - :noindex: - :undoc-members: - :show-inheritance: - -`Atmospheric correction` ------------------------- - -.. automodule:: SIAC.the_correction - :members: atmospheric_correction - :noindex: - :undoc-members: - :show-inheritance: - diff --git a/docs/source/authors.rst b/docs/source/authors.rst deleted file mode 100644 index da35d87..0000000 --- a/docs/source/authors.rst +++ /dev/null @@ -1,2 +0,0 @@ -.. _authors: - .. include:: ../../AUTHORS.rst diff --git a/docs/source/changes.rst b/docs/source/changes.rst deleted file mode 100644 index 0eb2568..0000000 --- a/docs/source/changes.rst +++ /dev/null @@ -1,2 +0,0 @@ -.. _changes: - .. include:: ../../CHANGES.md diff --git a/docs/source/conf.py b/docs/source/conf.py deleted file mode 100644 index 9e45929..0000000 --- a/docs/source/conf.py +++ /dev/null @@ -1,203 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Configuration file for the Sphinx documentation builder. -# -# This file does only contain a selection of the most common options. For a -# full list see the documentation: -# http://www.sphinx-doc.org/en/master/config - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys -sys.path.insert(0, os.path.abspath('../..')) - -user_pass = 'a', 'b' -with open('../../SIAC/data/.earthdata_auth', 'wb') as f: - for i in user_pass: - f.write((i+'\n').encode()) -import SIAC -with open('../../SIAC/data/.earthdata_auth', 'wb') as f: - f.write(('').encode()) - -# -- Project information ----------------------------------------------------- - -project = 'SIAC' -copyright = '2018, Feng Yin' -author = 'Feng Yin' - -# The short X.Y version -# The full version, including alpha/beta/rc tags -release = '2.1.1' - - -# -- General configuration --------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] - -source_suffix = ['.rst'] -#source_suffix = '.md' - -# The master toctree document. -master_doc = 'index' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = None - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'sphinx_rtd_theme' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# The default sidebars (for documents that don't match any pattern) are -# defined by theme itself. Builtin themes are using these templates by -# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', -# 'searchbox.html']``. -# -# html_sidebars = {} - - -# -- Options for HTMLHelp output --------------------------------------------- - -# Output file base name for HTML help builder. -htmlhelp_basename = 'SIACdoc' - - -# -- Options for LaTeX output ------------------------------------------------ - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'SIAC.tex', 'SIAC Documentation', - 'Feng Yin', 'manual'), -] - - -# -- Options for manual page output ------------------------------------------ - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'siac', 'SIAC Documentation', - [author], 1) -] - - -# -- Options for Texinfo output ---------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'SIAC', 'SIAC Documentation', - author, 'SIAC', 'One line description of project.', - 'Miscellaneous'), -] - - -# -- Options for Epub output ------------------------------------------------- - -# Bibliographic Dublin Core info. -epub_title = project - -# The unique identifier of the text. This can be a ISBN number -# or the project homepage. -# -# epub_identifier = '' - -# A unique identification for the text. -# -# epub_uid = '' - -# A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] - - -# -- Extension configuration ------------------------------------------------- - -# -- Options for intersphinx extension --------------------------------------- - -# Example configuration for intersphinx: refer to the Python standard library. -#intersphinx_mapping = {'https://docs.python.org/': None} - -# -- Options for todo extension ---------------------------------------------- - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = True diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index 03e74d9..0000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,55 +0,0 @@ -.. SIAC documentation master file, created by - sphinx-quickstart on Thu Nov 8 18:01:40 2018. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -MULTIPLY - Sensor Invariant Atmospheric Correction (SIAC) -========================================================= - -This atmospheric correction method uses MODIS MCD43 BRDF product to get -a coarse resolution simulation of earth surface. A model based on MODIS -PSF is built to deal with the scale differences between MODIS and other -sensors, and linear spectral mapping is used to map between different -sensors spectrally. We uses the ECMWF -`CAMS `__ -prediction as a prior for the atmospheric states, coupling with 6S model -to solve for the atmospheric parameters, then the solved atmospheric -parameters are used to correct the TOA reflectances. The whole system is -built under Bayesian theory and the uncertainty is propagated through -the whole system. Since we do not rely on specific bands' relationship -to estimate the atmospheric states, but instead a more generic and -consistent way of inversion those parameters. The code can be downloaded -from `SIAC `__ github -directly and futrher updates will make it more independent and can be -installed on different machines. - - - -Development of this code has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 687320, under project H2020 MULTIPLY. -f this code has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 687320, under project `H2020 MULTIPLY `_. - - - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - 01_Introduction.rst - 02_Installation.rst - 03_Quickstart.rst - 04_Data_access_and_adaption_for_Landsat5.rst - 05_Annex_technical.rst - -.. toctree:: - :maxdepth: 2 - :caption: anything else - - Authors - Changelog - -Indices and tables ------------------- - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/source/requirements.txt b/docs/source/requirements.txt deleted file mode 100644 index b9d419a..0000000 --- a/docs/source/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -sphinxcontrib-napoleon diff --git a/environment.yml b/environment.yml index bcacb87..e051e3c 100644 --- a/environment.yml +++ b/environment.yml @@ -5,7 +5,7 @@ channels: dependencies: - python=3.6* - - gdal=2.3.3 + - gdal=2.2 - numpy>=1.13 - scipy=1.0 - psutil diff --git a/requirements.txt b/requirements.txt index d58e486..4942247 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,8 @@ psutil requests scikit-image scikit-learn +GDAL>=2.1, <2.3 gp_emulator>=1.6.1 lightgbm>=2.1.0 numpy>=1.13 scipy>=1.0.0 -gdal>=2.1 diff --git a/setup.py b/setup.py index 0cfbe65..8c2afb4 100644 --- a/setup.py +++ b/setup.py @@ -1,18 +1,10 @@ -import os from setuptools import setup -file_path = os.path.dirname(os.path.realpath(__file__)) - -try: - version = os.environ['SIAC_VERSION'] -except: - version_file = open(os.path.join(file_path, 'SIAC/VERSION'), 'rb') - version = version_file.read().decode().strip() with open('README.md', 'rb') as f: readme = f.read().decode() -setup(name = 'SIAC', - version = version, +setup(name ='SIAC', + version ='2.0.9', description = 'A sensor invariant Atmospheric Correction (SIAC)', long_description = readme, long_description_content_type ='text/markdown', @@ -21,7 +13,7 @@ classifiers = ['Development Status :: 4 - Beta', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.6'], - install_requires = ['gdal>=2.1', 'numpy>=1.13', 'scipy>=1.0', 'psutil','six', + install_requires = ['gdal>=2.1, <2.3', 'numpy>=1.13', 'scipy>=1.0', 'psutil','six', 'lightgbm>=2.1.0','requests', 'scikit-learn', 'scikit-image', 'gp_emulator'], url = 'https://github.com/MarcYin/SIAC', license = "GNU Affero General Public License v3.0", From 786129fbd2713953013f81a5de6628c778d501ec Mon Sep 17 00:00:00 2001 From: TonioF Date: Wed, 10 Apr 2019 10:41:13 +0200 Subject: [PATCH 103/103] consider case when all data is invalid --- SIAC/the_aerosol.py | 2 ++ SIAC/the_correction.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/SIAC/the_aerosol.py b/SIAC/the_aerosol.py index 4def201..db47b65 100644 --- a/SIAC/the_aerosol.py +++ b/SIAC/the_aerosol.py @@ -644,6 +644,8 @@ def _fill_nan(self,): def fill_nan(array): x_shp, y_shp = array.shape mask = ~np.isnan(array) + if np.sum(mask) == 0: # no valid data + return np.full((x_shp, y_shp), np.nan) valid = np.array(np.where(mask)).T value = array[mask] mesh = np.repeat(range(x_shp), y_shp).reshape(x_shp, y_shp), \ diff --git a/SIAC/the_correction.py b/SIAC/the_correction.py index 830d8ea..496943c 100644 --- a/SIAC/the_correction.py +++ b/SIAC/the_correction.py @@ -311,8 +311,10 @@ def _parse_aux(self,): def _fill_nan(self,): def fill_nan(array): - x_shp, y_shp = array.shape - mask = ~np.isnan(array) + x_shp, y_shp = array.shape + mask = ~np.isnan(array) + if np.sum(mask) == 0: # all data is invalid + return np.full((x_shp, y_shp), np.nan) valid = np.array(np.where(mask)).T value = array[mask] mesh = np.repeat(range(x_shp), y_shp).reshape(x_shp, y_shp), \